gha: vfio: Import jobs scripts from tests repo

This imports the vfio test scripts github.com/kata-containers/tests. The test case doesn't work yet but doing the changes in a separate commit will make it easier to track the changes. The only change in this commit is renaming vfio_jenkins_job_build.sh -> vfio_fedora_vm_wrapper.sh Fixes: #6555 Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
2025-04-29 12:14:48 +00:00 · 2023-08-21 16:06:04 +02:00 · 2023-08-21 16:06:04 +02:00 · 5a551a85b1
commit 5a551a85b1
parent a1e3fa7ac4
5 changed files with 1025 additions and 0 deletions
--- a/tests/functional/vfio/gha-run.sh
+++ b/tests/functional/vfio/gha-run.sh
@ -19,6 +19,7 @@ function install_dependencies() {

 function run() {
 	info "Running cri-containerd tests using ${KATA_HYPERVISOR} hypervisor"
+	"${vfio_dir}"/vfio_fedora_vm_wrapper.sh
 }

 function main() {
--- a/tests/functional/vfio/guest-kernel.json.in
+++ b/tests/functional/vfio/guest-kernel.json.in
@ -0,0 +1,176 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+{
+	"ociVersion": "1.0.0-rc2-dev",
+	"platform": {
+		"os": "linux",
+		"arch": "amd64"
+	},
+	"annotations": {
+		"io.katacontainers.config.hypervisor.enable_iommu": "false",
+		"io.katacontainers.config.runtime.vfio_mode": "guest-kernel"
+	},
+	"process": {
+		"terminal": false,
+		"consoleSize": {
+			"height": 0,
+			"width": 0
+		},
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [ "/bin/tail", "-f", "/dev/null" ],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"rlimits": [{
+			"type": "RLIMIT_NOFILE",
+			"hard": 1024,
+			"soft": 1024
+		}],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "@ROOTFS@",
+		"readonly": false
+	},
+	"hostname": "vfio-test",
+	"mounts": [{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"hooks": {},
+	"linux": {
+		"devices": [{
+			"path": "@VFIO_PATH@",
+			"type": "c",
+			"major": @VFIO_MAJOR@,
+			"minor": @VFIO_MINOR@,
+			"fileMode": 384,
+			"uid": 0,
+			"gid": 0
+		}],
+		"cgroupsPath": "kata/vfiotest",
+		"resources": {
+			"devices": [
+				{"allow":false,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":3,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":5,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":8,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":9,"access":"rwm"},
+				{"allow":true,"type":"c","major":5,"minor":0,"access":"rwm"},
+				{"allow":true,"type":"c","major":5,"minor":1,"access":"rwm"},
+				{"allow": true,"access": "rwm","major": @VFIO_MAJOR@,"minor": @VFIO_MINOR@,"type": "c"}
+			]
+		},
+		"namespaces": [{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/kcore",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware"
+		],
+		"readonlyPaths": [
+			"/proc/asound",
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}
--- a/tests/functional/vfio/run.sh
+++ b/tests/functional/vfio/run.sh
@ -0,0 +1,341 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -x
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+script_path=$(dirname "$0")
+source "${script_path}/../../common.bash"
+
+addr=
+tmp_data_dir="$(mktemp -d)"
+rootfs_tar="${tmp_data_dir}/rootfs.tar"
+trap cleanup EXIT
+
+# kata-runtime options
+SANDBOX_CGROUP_ONLY=""
+HYPERVISOR=
+MACHINE_TYPE=
+IMAGE_TYPE=
+
+cleanup() {
+	clean_env_ctr
+	sudo rm -rf "${tmp_data_dir}"
+
+	[ -n "${host_pci}" ] && sudo driverctl unset-override "${host_pci}"
+}
+
+host_pci_addr() {
+	lspci -D | grep "Ethernet controller" | grep "Virtio.*network device" | tail -1 | cut -d' ' -f1
+}
+
+get_vfio_path() {
+	local addr="$1"
+	echo "/dev/vfio/$(basename $(realpath /sys/bus/pci/drivers/vfio-pci/${host_pci}/iommu_group))"
+}
+
+pull_rootfs() {
+	# pull and export busybox image in tar file
+	local image="quay.io/prometheus/busybox:latest"
+	sudo -E ctr i pull ${image}
+	sudo -E ctr i export "${rootfs_tar}" "${image}"
+	sudo chown ${USER}:${USER} "${rootfs_tar}"
+	sync
+}
+
+create_bundle() {
+	local bundle_dir="$1"
+	mkdir -p "${bundle_dir}"
+
+	# extract busybox rootfs
+	local rootfs_dir="${bundle_dir}/rootfs"
+	mkdir -p "${rootfs_dir}"
+	local layers_dir="$(mktemp -d)"
+	tar -C "${layers_dir}" -pxf "${rootfs_tar}"
+	for ((i=0;i<$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers | length");i++)); do
+		tar -C ${rootfs_dir} -xf ${layers_dir}/$(cat ${layers_dir}/manifest.json | jq -r ".[].Layers[${i}]")
+	done
+	sync
+
+	# Copy config.json
+	cp -a "${script_path}/config.json" "${bundle_dir}/config.json"
+}
+
+run_container() {
+	local container_id="$1"
+	local bundle_dir="$2"
+
+	sudo -E ctr run -d --runtime io.containerd.kata.v2 --config "${bundle_dir}/config.json" "${container_id}"
+}
+
+
+get_ctr_cmd_output() {
+	local container_id="$1"
+	shift
+	sudo -E ctr t exec --exec-id 2 "${container_id}" "${@}"
+}
+
+check_guest_kernel() {
+	local container_id="$1"
+	# For vfio_mode=guest-kernel, the device should be bound to
+	# the guest kernel's native driver.  To check this has worked,
+	# we look for an ethernet device named 'eth*'
+	get_ctr_cmd_output "${container_id}" ip a | grep "eth" || die "Missing VFIO network interface"
+}
+
+check_vfio() {
+	local cid="$1"
+	# For vfio_mode=vfio, the device should be bound to the guest
+	# vfio-pci driver.
+
+	# Check the control device is visible
+	get_ctr_cmd_output "${cid}" ls /dev/vfio/vfio || die "Couldn't find VFIO control device in container"
+
+	# The device should *not* cause an ethernet interface to appear
+	! get_ctr_cmd_output "${cid}" ip a | grep "eth" || die "Unexpected network interface"
+
+	# There should be exactly one VFIO group device (there might
+	# be multiple IOMMU groups in the VM, but only one device
+	# should be bound to the VFIO driver, so there should still
+	# only be one VFIO device
+	group="$(get_ctr_cmd_output "${cid}" ls /dev/vfio | grep -v vfio)"
+	if [ $(echo "${group}" | wc -w) != "1" ] ; then
+	    die "Expected exactly one VFIO group got: ${group}"
+	fi
+
+	# There should be two devices in the IOMMU group: the ethernet
+	# device we care about, plus the PCIe to PCI bridge device
+	devs="$(get_ctr_cmd_output "${cid}" ls /sys/kernel/iommu_groups/"${group}"/devices)"
+	if [ $(echo "${devs}" | wc -w) != "2" ] ; then
+	    die "Expected exactly two devices got: ${devs}"
+	fi
+
+	# The bridge device will always sort first, because it is on
+	# bus zero, whereas the NIC will be on a non-zero bus
+	guest_pci=$(echo "${devs}" | tail -1)
+
+	# This is a roundabout way of getting the environment
+	# variable, but to use the more obvious "echo $PCIDEVICE_..."
+	# we would have to escape the '$' enough to not be expanded
+	# before it's injected into the container, but not so much
+	# that it *is* expanded by the shell within the container.
+	# Doing that with another shell function in between is very
+	# fragile, so do it this way instead.
+	guest_env="$(get_ctr_cmd_output "${cid}" env | grep ^PCIDEVICE_VIRTIO_NET | sed s/^[^=]*=//)"
+	if [ "${guest_env}" != "${guest_pci}" ]; then
+	    die "PCIDEVICE variable was \"${guest_env}\" instead of \"${guest_pci}\""
+	fi
+}
+
+get_dmesg() {
+	local container_id="$1"
+	get_ctr_cmd_output "${container_id}" dmesg
+}
+
+# Show help about this script
+help(){
+cat << EOF
+Usage: $0 [-h] [options]
+    Description:
+        This script runs a kata container and passthrough a vfio device
+    Options:
+        -h,          Help
+        -i <string>, Specify initrd or image
+        -m <string>, Specify kata-runtime machine type for qemu hypervisor
+        -p <string>, Specify kata-runtime hypervisor
+        -s <value>,  Set sandbox_cgroup_only in the configuration file
+EOF
+}
+
+setup_configuration_file() {
+	local qemu_config_file="configuration-qemu.toml"
+	local clh_config_file="configuration-clh.toml"
+	local image_file="/opt/kata/share/kata-containers/kata-containers.img"
+	local initrd_file="/opt/kata/share/kata-containers/kata-containers-initrd.img"
+	local kata_config_file=""
+
+	for file in $(kata-runtime --kata-show-default-config-paths); do
+		if [ ! -f "${file}" ]; then
+			continue
+		fi
+
+		kata_config_file="${file}"
+		config_dir=$(dirname ${file})
+		config_filename=""
+
+		if [ "$HYPERVISOR" = "qemu" ]; then
+			config_filename="${qemu_config_file}"
+		elif [ "$HYPERVISOR" = "clh" ]; then
+			config_filename="${clh_config_file}"
+		fi
+
+		config_file="${config_dir}/${config_filename}"
+		if [ -f "${config_file}" ]; then
+			rm -f "${kata_config_file}"
+			cp -a $(realpath "${config_file}") "${kata_config_file}"
+			break
+		fi
+	done
+
+	# machine type applies to configuration.toml and configuration-qemu.toml
+	if [ -n "$MACHINE_TYPE" ]; then
+		if [ "$HYPERVISOR" = "qemu" ]; then
+			sed -i 's|^machine_type.*|machine_type = "'${MACHINE_TYPE}'"|g' "${kata_config_file}"
+			# Make sure we have set hot_plug_vfio to a reasonable value
+			sudo sed -i -e 's|^#hot_plug_vfio =.*$|hot_plug_vfio = "bridge-port"|' -e 's|^hot_plug_vfio = .*$|hot_plug_vfio = "bridge-port"|' "${kata_config_file}"
+		else
+			warn "Variable machine_type only applies to qemu. It will be ignored"
+		fi
+	fi
+
+	if [ -n "${SANDBOX_CGROUP_ONLY}" ]; then
+	   sed -i 's|^sandbox_cgroup_only.*|sandbox_cgroup_only='${SANDBOX_CGROUP_ONLY}'|g' "${kata_config_file}"
+	fi
+
+	# Change to initrd or image depending on user input.
+	# Non-default configs must be changed to specify either initrd or image, image is default.
+	if [ "$IMAGE_TYPE" = "initrd" ]; then
+		if $(grep -q "^image.*" ${kata_config_file}); then
+			if $(grep -q "^initrd.*" ${kata_config_file}); then
+				sed -i '/^image.*/d' "${kata_config_file}"
+			else
+				sed -i 's|^image.*|initrd = "'${initrd_file}'"|g' "${kata_config_file}"
+			fi
+		fi
+	else
+		if $(grep -q "^initrd.*" ${kata_config_file}); then
+			if $(grep -q "^image.*" ${kata_config_file}); then
+				sed -i '/^initrd.*/d' "${kata_config_file}"
+			else
+				sed -i 's|^initrd.*|image = "'${image_file}'"|g' "${kata_config_file}"
+			fi
+		fi
+	fi
+
+	# enable debug
+	sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' \
+	       -e 's/^#\(debug_console_enabled\).*=.*$/\1 = true/g' \
+	       -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug"/g' \
+	       "${kata_config_file}"
+
+	# enable VFIO relevant hypervisor annotations
+	sed -i -e 's/^\(enable_annotations\).*=.*$/\1 = ["enable_iommu"]/' \
+		"${kata_config_file}"
+}
+
+run_test_container() {
+	local container_id="$1"
+	local bundle_dir="$2"
+	local config_json_in="$3"
+	local host_pci="$4"
+
+	# generate final config.json
+	sed -e '/^#.*/d' \
+	    -e 's|@VFIO_PATH@|'"${vfio_device}"'|g' \
+	    -e 's|@VFIO_MAJOR@|'"${vfio_major}"'|g' \
+	    -e 's|@VFIO_MINOR@|'"${vfio_minor}"'|g' \
+	    -e 's|@VFIO_CTL_MAJOR@|'"${vfio_ctl_major}"'|g' \
+	    -e 's|@VFIO_CTL_MINOR@|'"${vfio_ctl_minor}"'|g' \
+	    -e 's|@ROOTFS@|'"${bundle_dir}/rootfs"'|g' \
+	    -e 's|@HOST_PCI@|'"${host_pci}"'|g' \
+	    "${config_json_in}" > "${script_path}/config.json"
+
+	create_bundle "${bundle_dir}"
+
+	# run container
+	run_container "${container_id}" "${bundle_dir}"
+
+	# output VM dmesg
+	get_dmesg "${container_id}"
+}
+
+main() {
+	local OPTIND
+	while getopts "hi:m:p:s:" opt;do
+		case ${opt} in
+		h)
+		    help
+		    exit 0;
+		    ;;
+		i)
+		    IMAGE_TYPE="${OPTARG}"
+		    ;;
+		m)
+		    MACHINE_TYPE="${OPTARG}"
+		    ;;
+		p)
+		    HYPERVISOR="${OPTARG}"
+		    ;;
+		s)
+		    SANDBOX_CGROUP_ONLY="${OPTARG}"
+		    ;;
+		?)
+		    # parse failure
+		    help
+		    die "Failed to parse arguments"
+		    ;;
+		esac
+	done
+	shift $((OPTIND-1))
+
+	#
+	# Get the device ready on the host
+	#
+	setup_configuration_file
+
+	restart_containerd_service
+	sudo modprobe vfio
+	sudo modprobe vfio-pci
+
+	host_pci=$(host_pci_addr)
+	[ -n "${host_pci}" ] || die "virtio ethernet controller PCI address not found"
+
+	cat /proc/cmdline | grep -q "intel_iommu=on" || \
+		die "intel_iommu=on not found in kernel cmdline"
+
+	sudo driverctl set-override "${host_pci}" vfio-pci
+
+	vfio_device="$(get_vfio_path "${host_pci}")"
+	[ -n "${vfio_device}" ] || die "vfio device not found"
+	vfio_major="$(printf '%d' $(stat -c '0x%t' ${vfio_device}))"
+	vfio_minor="$(printf '%d' $(stat -c '0x%T' ${vfio_device}))"
+
+	[ -n "/dev/vfio/vfio" ] || die "vfio control device not found"
+	vfio_ctl_major="$(printf '%d' $(stat -c '0x%t' /dev/vfio/vfio))"
+	vfio_ctl_minor="$(printf '%d' $(stat -c '0x%T' /dev/vfio/vfio))"
+
+	# Get the rootfs we'll use for all tests
+	pull_rootfs
+
+	#
+	# Run the tests
+	#
+
+	# test for guest-kernel mode
+	guest_kernel_cid="vfio-guest-kernel-${RANDOM}"
+	run_test_container "${guest_kernel_cid}" \
+			   "${tmp_data_dir}/vfio-guest-kernel" \
+			   "${script_path}/guest-kernel.json.in" \
+			   "${host_pci}"
+	check_guest_kernel "${guest_kernel_cid}"
+
+	# Remove the container so we can re-use the device for the next test
+	clean_env_ctr
+
+	# test for vfio mode
+	vfio_cid="vfio-vfio-${RANDOM}"
+	run_test_container "${vfio_cid}" \
+			   "${tmp_data_dir}/vfio-vfio" \
+			   "${script_path}/vfio.json.in" \
+			   "${host_pci}"
+	check_vfio "${vfio_cid}"
+}
+
+main $@
--- a/tests/functional/vfio/vfio.json.in
+++ b/tests/functional/vfio/vfio.json.in
@ -0,0 +1,187 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+{
+	"ociVersion": "1.0.0-rc2-dev",
+	"platform": {
+		"os": "linux",
+		"arch": "amd64"
+	},
+	"annotations": {
+		"io.katacontainers.config.hypervisor.enable_iommu": "true",
+		"io.katacontainers.config.runtime.vfio_mode": "vfio"
+	},
+	"process": {
+		"terminal": false,
+		"consoleSize": {
+			"height": 0,
+			"width": 0
+		},
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [ "/bin/tail", "-f", "/dev/null" ],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm",
+			"PCIDEVICE_VIRTIO_NET=@HOST_PCI@"
+		],
+		"cwd": "/",
+		"rlimits": [{
+			"type": "RLIMIT_NOFILE",
+			"hard": 1024,
+			"soft": 1024
+		}],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "@ROOTFS@",
+		"readonly": false
+	},
+	"hostname": "vfio-test",
+	"mounts": [{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"hooks": {},
+	"linux": {
+		"devices": [{
+			"path": "/dev/vfio/vfio",
+			"type": "c",
+			"major": @VFIO_CTL_MAJOR@,
+			"minor": @VFIO_CTL_MINOR@,
+			"fileMode": 438,
+			"uid": 0,
+			"gid": 0
+		},
+		{
+			"path": "@VFIO_PATH@",
+			"type": "c",
+			"major": @VFIO_MAJOR@,
+			"minor": @VFIO_MINOR@,
+			"fileMode": 384,
+			"uid": 0,
+			"gid": 0
+		}],
+		"cgroupsPath": "kata/vfiotest",
+		"resources": {
+			"devices": [
+				{"allow":false,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":3,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":5,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":8,"access":"rwm"},
+				{"allow":true,"type":"c","major":1,"minor":9,"access":"rwm"},
+				{"allow":true,"type":"c","major":5,"minor":0,"access":"rwm"},
+				{"allow":true,"type":"c","major":5,"minor":1,"access":"rwm"},
+				{"allow": true,"access": "rwm","major": @VFIO_CTL_MAJOR@,"minor": @VFIO_CTL_MINOR@,"type": "c"},
+				{"allow": true,"access": "rwm","major": @VFIO_MAJOR@,"minor": @VFIO_MINOR@,"type": "c"}
+			]
+		},
+		"namespaces": [{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/kcore",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware"
+		],
+		"readonlyPaths": [
+			"/proc/asound",
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}
--- a/tests/functional/vfio/vfio_fedora_vm_wrapper.sh
+++ b/tests/functional/vfio/vfio_fedora_vm_wrapper.sh
@ -0,0 +1,320 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Run the .ci/jenkins_job_build.sh script in a VM
+# that supports VFIO, then run VFIO functional tests
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+cidir=$(dirname "$0")
+
+source /etc/os-release || source /usr/lib/os-release
+# <CHANGES HERE>
+#source "${cidir}/lib.sh"
+export WORKSPACE="${WORKSPACE:-${cidir}/../../..}"
+export GIT_URL="https://github.com/kata-containers/kata-containers.git"
+# </CHANGES>
+
+http_proxy=${http_proxy:-}
+https_proxy=${https_proxy:-}
+vm_ip="127.0.15.1"
+vm_port="10022"
+# Don't save data in /tmp, we need it after rebooting the system
+data_dir="${HOME}/functional-vfio-test"
+ssh_key_file="${data_dir}/key"
+arch=$(uname -m)
+artifacts_dir="${WORKSPACE}/artifacts"
+
+kill_vms() {
+	sudo killall -9 qemu-system-${arch}
+}
+
+cleanup() {
+	mkdir -p ${artifacts_dir}
+	sudo chown -R ${USER} ${artifacts_dir}
+	scp_vm ${artifacts_dir}/* ${artifacts_dir} || true
+	kill_vms
+}
+
+create_ssh_key() {
+	rm -f "${ssh_key_file}"
+	ssh-keygen -f "${ssh_key_file}" -t rsa -N ""
+}
+
+create_meta_data() {
+	file="$1"
+	cat <<EOF > "${file}"
+{
+  "uuid": "d1b4aafa-5d75-4f9c-87eb-2ceabe110c39",
+  "hostname": "test"
+}
+EOF
+}
+
+create_user_data() {
+	file="$1"
+	ssh_pub_key_file="$2"
+
+	ssh_pub_key="$(cat "${ssh_pub_key_file}")"
+	dnf_proxy=""
+	service_proxy=""
+	docker_user_proxy="{}"
+	environment=$(env | egrep "ghprb|WORKSPACE|KATA|GIT|JENKINS|_PROXY|_proxy" | \
+	                    sed -e "s/'/'\"'\"'/g" \
+	                        -e "s/\(^[[:alnum:]_]\+\)=/\1='/" \
+	                        -e "s/$/'/" \
+	                        -e 's/^/    export /')
+
+	if [ -n "${http_proxy}" ] && [ -n "${https_proxy}" ]; then
+		dnf_proxy="proxy=${http_proxy}"
+		service_proxy='[Service]
+    Environment="HTTP_PROXY='${http_proxy}'" "HTTPS_PROXY='${https_proxy}'" "NO_PROXY='${no_proxy}'"'
+		docker_user_proxy='{"proxies": { "default": {
+    "httpProxy": "'${http_proxy}'",
+    "httpsProxy": "'${https_proxy}'",
+    "noProxy": "'${no_proxy}'"
+    } } }'
+	fi
+
+	cat <<EOF > "${file}"
+#cloud-config
+package_upgrade: false
+runcmd:
+- chown -R ${USER}:${USER} /home/${USER}
+- touch /.done
+users:
+- gecos: User
+  gid: "1000"
+  lock-passwd: true
+  name: ${USER}
+  shell: /bin/bash
+  ssh-authorized-keys:
+  - ${ssh_pub_key}
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  uid: "1000"
+write_files:
+- content: |
+${environment}
+  path: /etc/environment
+- content: |
+    ${service_proxy}
+  path: /etc/systemd/system/docker.service.d/http-proxy.conf
+- content: |
+    ${service_proxy}
+  path: /etc/systemd/system/containerd.service.d/http-proxy.conf
+- content: |
+    ${docker_user_proxy}
+  path: ${HOME}/.docker/config.json
+- content: |
+    ${docker_user_proxy}
+  path: /root/.docker/config.json
+- content: |
+    set -x
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+    set -o errtrace
+    . /etc/environment
+    . /etc/os-release
+
+    [ "\$ID" = "fedora" ] || (echo >&2 "$0 only supports Fedora"; exit 1)
+
+    echo "${dnf_proxy}" | sudo tee -a /etc/dnf/dnf.conf
+
+    for i in \$(seq 1 50); do
+        [ -f /.done ] && break
+        echo "waiting for cloud-init to finish"
+        sleep 5;
+    done
+
+    export FORCE_JENKINS_JOB_BUILD=1
+    export DEBUG=true
+    export CI_JOB="VFIO"
+    export GOPATH=\${WORKSPACE}/go
+    export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:\${PATH}
+    export GOROOT="/usr/local/go"
+    export KUBERNETES="no"
+    export USE_DOCKER="true"
+    export ghprbPullId
+    export ghprbTargetBranch
+
+    # Make sure the packages were installed
+    # Sometimes cloud-init is unable to install them
+    sudo dnf makecache
+    sudo dnf install -y git make pciutils driverctl
+
+    git config --global user.email "foo@bar"
+    git config --global user.name "Foo Bar"
+
+    tests_repo="github.com/kata-containers/tests"
+    tests_repo_dir="\${GOPATH}/src/\${tests_repo}"
+    trap "cd \${tests_repo_dir}; sudo -E PATH=\$PATH .ci/teardown.sh ${artifacts_dir} || true; sudo chown -R \${USER} ${artifacts_dir}" EXIT
+
+    curl -OL https://raw.githubusercontent.com/kata-containers/tests/\${ghprbTargetBranch}/.ci/ci_entry_point.sh
+    bash -x ci_entry_point.sh "${GIT_URL}"
+
+  path: /home/${USER}/run.sh
+  permissions: '0755'
+EOF
+}
+
+create_config_iso() {
+	iso_file="$1"
+	ssh_pub_key_file="${ssh_key_file}.pub"
+	iso_data_dir="${data_dir}/d"
+	meta_data_file="${iso_data_dir}/openstack/latest/meta_data.json"
+	user_data_file="${iso_data_dir}/openstack/latest/user_data"
+
+	mkdir -p $(dirname "${user_data_file}")
+
+	create_meta_data "${meta_data_file}"
+	create_user_data "${user_data_file}" "${ssh_pub_key_file}"
+
+	[ -f "${iso_file}" ] && rm -f "${iso_file}"
+
+	xorriso -as mkisofs -R -V config-2 -o "${iso_file}" "${iso_data_dir}"
+}
+
+pull_fedora_cloud_image() {
+	fedora_img="$1"
+	fedora_version=37
+	# Add a version to the image cache, otherwise the tests are going to 
+	# use always the same image without rebuilding it, regardless the version
+	# set in fedora_version
+	fedora_img_cache="${fedora_img}.cache.${fedora_version}"
+	fedora_img_url="https://download.fedoraproject.org/pub/fedora/linux/releases/${fedora_version}/Cloud/${arch}/images/Fedora-Cloud-Base-${fedora_version}-1.7.${arch}.raw.xz"
+
+	if [ ! -f "${fedora_img_cache}" ]; then
+		curl -sL ${fedora_img_url} -o "${fedora_img_cache}.xz"
+		xz -f -d "${fedora_img_cache}.xz"
+		sync
+	fi
+
+	cp -a "${fedora_img_cache}" "${fedora_img}"
+	sync
+
+	# setup cloud image
+	sudo losetup -D
+	loop=$(sudo losetup --show -Pf "${fedora_img}")
+	sudo mount "${loop}p2" /mnt
+
+	# add intel_iommu=on to the guest kernel command line
+	kernelopts="intel_iommu=on iommu=pt selinux=0"
+	entries=$(sudo ls /mnt/loader/entries/)
+	for entry in ${entries}; do
+		sudo sed -i '/^options /  s/$/ intel_iommu=on iommu=pt selinux=0 /g' /mnt/loader/entries/"${entry}"
+	done
+	sudo sed -i 's|kernelopts="|kernelopts="'"${kernelopts}"'|g' /mnt/grub2/grub.cfg
+	sudo sed -i 's|kernelopts=|kernelopts='"${kernelopts}"'|g' /mnt/grub2/grubenv
+
+	# cleanup
+	sudo umount -R /mnt/
+	sudo losetup -d "${loop}"
+
+	qemu-img resize -f raw "${fedora_img}" +20G
+}
+
+run_vm() {
+	image="$1"
+	config_iso="$2"
+	disable_modern="off"
+	hostname="$(hostname)"
+	memory="16384M"
+	cpus=$(nproc)
+	machine_type="q35"
+
+	/usr/bin/qemu-system-${arch} -m "${memory}" -smp cpus="${cpus}" \
+	   -cpu host,host-phys-bits \
+	   -machine ${machine_type},accel=kvm,kernel_irqchip=split \
+	   -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on \
+	   -drive file=${image},if=virtio,aio=threads,format=raw \
+	   -drive file=${config_iso_file},if=virtio,media=cdrom \
+	   -daemonize -enable-kvm -device virtio-rng-pci -display none -vga none \
+	   -netdev user,hostfwd=tcp:${vm_ip}:${vm_port}-:22,hostname="${hostname}",id=net0 \
+	   -device virtio-net-pci,netdev=net0,disable-legacy=on,disable-modern="${disable_modern}",iommu_platform=on,ats=on \
+	   -netdev user,id=net1 \
+	   -device virtio-net-pci,netdev=net1,disable-legacy=on,disable-modern="${disable_modern}",iommu_platform=on,ats=on
+}
+
+install_dependencies() {
+	case "${ID}" in
+		ubuntu)
+			# cloud image dependencies
+			deps=(xorriso curl qemu-utils openssh-client)
+
+			sudo apt-get update
+			sudo apt-get install -y ${deps[@]}
+			sudo apt-get install --reinstall -y qemu-system-x86
+			;;
+		fedora)
+			# cloud image dependencies
+			deps=(xorriso curl qemu-img openssh)
+
+			sudo dnf install -y ${deps[@]} qemu-system-x86-core
+			sudo dnf reinstall -y qemu-system-x86-core
+			;;
+
+		"*")
+			die "Unsupported distro: ${ID}"
+			;;
+	esac
+}
+
+ssh_vm() {
+	cmd=$@
+	ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "${ssh_key_file}" -p "${vm_port}" "${USER}@${vm_ip}" "${cmd}"
+}
+
+scp_vm() {
+	guest_src=$1
+	host_dest=$2
+	scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i "${ssh_key_file}" -P "${vm_port}" ${USER}@${vm_ip}:${guest_src} ${host_dest}
+}
+
+wait_for_vm() {
+	for i in $(seq 1 30); do
+		if ssh_vm true; then
+			return 0
+		fi
+		info "waiting for VM to start"
+		sleep 5
+	done
+	return 1
+}
+
+main() {
+	trap cleanup EXIT
+
+	config_iso_file="${data_dir}/config.iso"
+	fedora_img="${data_dir}/image.img"
+
+	mkdir -p "${data_dir}"
+
+	install_dependencies
+
+	create_ssh_key
+
+	create_config_iso "${config_iso_file}"
+
+	for i in $(seq 1 5); do
+		pull_fedora_cloud_image "${fedora_img}"
+		run_vm "${fedora_img}" "${config_iso_file}"
+		if wait_for_vm; then
+			break
+		fi
+		info "Couldn't connect to the VM. Stopping VM and starting a new one."
+		kill_vms
+	done
+
+	ssh_vm "/home/${USER}/run.sh"
+}
+
+main $@