CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #4275
Signed-off-by: Georgina Kinge <georgina.kinge@ibm.com>

.github/workflows/docs-url-alive-check.yaml

@@ -14,31 +14,31 @@ jobs:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
     - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     # docs url alive check
     - name: Docs URL Alive Check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      if: github.repository_owner == 'kata-containers'
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/kata-deploy-push.yaml

@@ -24,6 +24,7 @@ jobs:
           - firecracker
           - rootfs-image
           - rootfs-initrd
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/kata-deploy-test.yaml

@@ -47,6 +47,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
       - name: get-PR-ref
         id: get-PR-ref

.github/workflows/release.yaml

@@ -17,6 +17,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
+          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

ci/go-test.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2020 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-run_go_test

ci/lib.sh

@@ -39,12 +39,6 @@ run_static_checks()
 	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
-run_go_test()
-{
-	clone_tests_repo
-	bash "$tests_repo_dir/.ci/go-test.sh"
-}
-
 run_docs_url_alive_check()
 {
 	clone_tests_repo

docs/how-to/privileged.md

@@ -31,7 +31,7 @@ See below example config:
   [plugins.cri]
     [plugins.cri.containerd]
        [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v1"
+         runtime_type = "io.containerd.runc.v2"
          privileged_without_host_devices = false
        [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"

src/agent/src/rpc.rs

@@ -1755,7 +1755,7 @@ fn is_signal_handled(proc_status_file: &str, signum: u32) -> bool {
                 warn!(sl!(), "parse the SigCgt field failed");
                 return false;
             }
-            let sig_cgt_str = mask_vec[1];
+            let sig_cgt_str = mask_vec[1].trim();
             let sig_cgt_mask = match u64::from_str_radix(sig_cgt_str, 16) {
                 Ok(h) => h,
                 Err(_) => {
@@ -2534,6 +2534,26 @@ OtherField:other
                 signum: 4,
                 result: true,
             },
+            TestData {
+                status_file_data: Some("SigCgt:\t000000004b813efb"),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt: 000000004b813efb"),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt:000000004b813efb "),
+                signum: 4,
+                result: true,
+            },
+            TestData {
+                status_file_data: Some("SigCgt:\t000000004b813efb "),
+                signum: 4,
+                result: true,
+            },
             TestData {
                 status_file_data: Some("SigCgt:000000004b813efb"),
                 signum: 3,

src/runtime/.gitignore

@@ -2,6 +2,7 @@
 *.patch
 *.swp
 coverage.txt
+coverage.txt.tmp
 coverage.html
 .git-commit
 .git-commit.tmp

src/runtime/Makefile

@@ -176,6 +176,9 @@ DEFDISABLEBLOCK := false
 DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs
 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd
+ifeq ($(ARCH),amd64)
+DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd
+endif
 DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 # Default DAX mapping cache size in MiB
 #if value is 0, DAX is not enabled
@@ -596,7 +599,7 @@ hook:
 
 go-test: $(GENERATED_FILES)
 	go clean -testcache
-	$(QUIET_TEST)../../ci/go-test.sh
+	$(QUIET_TEST)./go-test.sh
 
 fast-test: $(GENERATED_FILES)
 	go clean -testcache

src/runtime/go-test.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+#
+# Copyright (c) 2017-2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+script_name=${0##*/}
+typeset -A long_options
+
+long_options=(
+	[help]="Show usage"
+	[package:]="Specify test package to run"
+)
+
+# Set up go test flags
+go_test_flags="${KATA_GO_TEST_FLAGS}"
+if [ -z "$go_test_flags" ]; then
+    # KATA_GO_TEST_TIMEOUT can be set to any value accepted by
+    # "go test -timeout X"
+    go_test_flags="-timeout ${KATA_GO_TEST_TIMEOUT:-30s}"
+
+    # -race flag is not supported on s390x
+    [ "$(go env GOARCH)" != "s390x" ] && go_test_flags+=" -race"
+
+    # s390x requires special linker flags
+    [ "$(go env GOARCH)" = s390x ] && go_test_flags+=" -ldflags '-extldflags -Wl,--s390-pgste'"
+fi
+
+# The "master" coverage file that contains the coverage results for
+# all packages run under all scenarios.
+test_coverage_file="coverage.txt"
+
+# Temporary coverage file created for a "go test" run. The results in
+# this file will be added to the master coverage file.
+tmp_coverage_file="${test_coverage_file}.tmp"
+
+warn()
+{
+	local msg="$*"
+	echo >&2 "WARNING: $msg"
+}
+
+usage()
+{
+	cat <<EOF
+
+Usage: $script_name [options]
+
+Options:
+
+EOF
+
+	local option
+	local description
+
+	local long_option_names="${!long_options[@]}"
+
+	# Sort space-separated list by converting to newline separated list
+	# and back again.
+	long_option_names=$(echo "$long_option_names"|tr ' ' '\n'|sort|tr '\n' ' ')
+
+	# Display long options
+	for option in ${long_option_names}
+	do
+		description=${long_options[$option]}
+
+		# Remove any trailing colon which is for getopt(1) alone.
+		option=$(echo "$option"|sed 's/:$//g')
+
+		printf "    --%-10.10s # %s\n" "$option" "$description"
+	done
+}
+
+# Run a command as either root or the current user (which might still be root).
+#
+# If the first argument is "root", run using sudo, else run as the current
+# user. All arguments after the first will be treated as the command to run.
+run_as_user()
+{
+	local user="$1"
+
+	shift
+
+	local cmd=$*
+
+	if [ "$user" = root ]; then
+		# use a shell to ensure PATH is correct.
+		sudo -E PATH="$PATH" sh -c "$cmd"
+	else
+		eval "$cmd"
+	fi
+}
+
+# Test a single golang package
+test_go_package()
+{
+	local -r pkg="$1"
+	local -r user="$2"
+
+	printf "INFO: Running 'go test' as %s user on package '%s' with flags '%s'\n" \
+		"$user" "$pkg" "$go_test_flags"
+
+	run_as_user "$user" go test "$go_test_flags" -covermode=atomic -coverprofile=$tmp_coverage_file "$pkg"
+
+	# Merge test results into the master coverage file.
+	run_as_user "$user" tail -n +2 "$tmp_coverage_file" >> "$test_coverage_file"
+	rm -f "$tmp_coverage_file"
+}
+
+# Run all tests and generate a test coverage file.
+test_coverage()
+{
+	echo "mode: atomic" > "$test_coverage_file"
+
+	users="current"
+
+	if [ "$(id -u)" -eq 0 ]; then
+		warn "Already running as root so will not re-run tests as non-root user."
+		warn "As a result, only a subset of tests will be run"
+		warn "(run this script as a non-privileged to ensure all tests are run)."
+	else
+		# Run the unit-tests *twice* (since some must run as
+		# root and others must run as non-root), combining the
+		# resulting test coverage files.
+		users+=" root"
+	fi
+
+	echo "INFO: Currently running as user '$(id -un)'"
+	for user in $users; do
+	    test_go_package "$package" "$user"
+	done
+}
+
+main()
+{
+	local long_option_names="${!long_options[@]}"
+
+	local args=$(getopt \
+		-n "$script_name" \
+		-a \
+		--options="h" \
+		--longoptions="$long_option_names" \
+		-- "$@")
+
+	package="./..."
+
+	eval set -- "$args"
+	[ $? -ne 0 ] && { usage >&2; exit 1; }
+
+	while [ $# -gt 1 ]
+	do
+		case "$1" in
+			-h|--help) usage; exit 0 ;;
+			--package) package="$2"; shift 2;;
+			--) shift; break ;;
+		esac
+
+		shift
+	done
+
+	test_coverage
+}
+
+main "$@"

src/runtime/virtcontainers/sandbox.go

@@ -1041,15 +1041,13 @@ func (cw *consoleWatcher) start(s *Sandbox) (err error) {
 		}
 
 		if err := scanner.Err(); err != nil {
-			if err == io.EOF {
+			s.Logger().WithError(err).WithFields(logrus.Fields{
-				s.Logger().Info("console watcher quits")
+				"console-protocol": cw.proto,
-			} else {
+				"console-url":      cw.consoleURL,
-				s.Logger().WithError(err).WithFields(logrus.Fields{
+				"sandbox":          s.id,
-					"console-protocol": cw.proto,
+			}).Error("Failed to read guest console logs")
-					"console-url":      cw.consoleURL,
+		} else { // The error is `nil` in case of io.EOF
-					"sandbox":          s.id,
+			s.Logger().Info("console watcher quits")
-				}).Error("Failed to read guest console logs")
-			}
 		}
 	}()
 

tools/packaging/kata-deploy/local-build/Makefile

@@ -27,7 +27,8 @@ all: cloud-hypervisor-tarball \
 	qemu-tarball \
 	rootfs-image-tarball \
 	rootfs-initrd-tarball \
-	shim-v2-tarball
+	shim-v2-tarball \
+	virtiofsd-tarball
 
 %-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh
 	$(call BUILD,$*)
@@ -57,6 +58,9 @@ rootfs-initrd-tarball:
 shim-v2-tarball:
 	${MAKE} $@-build
 
+virtiofsd-tarball:
+	${MAKE} $@-build
+
 merge-builds:
 	$(MK_DIR)/kata-deploy-merge-builds.sh build
 

tools/packaging/kata-deploy/local-build/dockerbuild/Dockerfile

@@ -35,6 +35,7 @@ RUN apt-get update && \
   gcc \
   git \
   make \
+  unzip \
   xz-utils && \
   apt-get clean && rm -rf /var/lib/apt/lists
 

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -26,6 +26,7 @@ readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firec
 readonly kernel_builder="${static_build_dir}/kernel/build.sh"
 readonly qemu_builder="${static_build_dir}/qemu/build-static-qemu.sh"
 readonly shimv2_builder="${static_build_dir}/shim-v2/build.sh"
+readonly virtiofsd_builder="${static_build_dir}/virtiofsd/build-static-virtiofsd.sh"
 
 readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_image.sh"
 
@@ -76,6 +77,7 @@ options:
 	rootfs-image
 	rootfs-initrd
 	shim-v2
+	virtiofsd
 EOF
 
 	exit "${return_code}"
@@ -140,6 +142,15 @@ install_clh() {
 	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor"
 }
 
+# Install static virtiofsd asset
+install_virtiofsd() {
+	info "build static virtiofsd"
+	"${virtiofsd_builder}"
+	info "Install static virtiofsd"
+	mkdir -p "${destdir}/opt/kata/libexec/"
+	sudo install -D --owner root --group root --mode 0744 virtiofsd/virtiofsd "${destdir}/opt/kata/libexec/virtiofsd"
+}
+
 #Install all components that are not assets
 install_shimv2() {
 	GO_VERSION="$(yq r ${versions_yaml} languages.golang.meta.newest-version)"
@@ -166,6 +177,7 @@ handle_build() {
 		install_kernel
 		install_qemu
 		install_shimv2
+		install_virtiofsd
 		;;
 
 	cloud-hypervisor) install_clh ;;
@@ -184,6 +196,8 @@ handle_build() {
 
 	shim-v2) install_shimv2 ;;
 
+	virtiofsd) install_virtiofsd ;;
+
 	*)
 		die "Invalid build target ${build_target}"
 		;;
@@ -221,6 +235,7 @@ main() {
 		rootfs-image
 		rootfs-initrd
 		shim-v2
+		virtiofsd
 	)
 	silent=false
 	while getopts "hs-:" opt; do

tools/packaging/kernel/configs/fragments/x86_64/nemu.conf

@@ -1,7 +0,0 @@
-# Items needed to run the NEMU cut of QEMU
-# NEMU uses an EFI bios/boot, so requires a few extra bits
-
-CONFIG_MSDOS_PARTITION=y
-CONFIG_EFI=y
-CONFIG_EFI_ESRT=y
-CONFIG_EFI_RUNTIME_WRAPPERS=y

tools/packaging/kernel/configs/fragments/x86_64/sgx.conf

@@ -1 +1,6 @@
+# Intel Software Guard Extensions
+
 CONFIG_X86_SGX=y
+
+# Requirement for Gramine Shielded Containers
+CONFIG_NUMA=y

tools/packaging/kernel/kata_config_version

@@ -1 +1 @@
-90
+92

tools/packaging/qemu/patches/6.2.x/0001-numa-Enable-numa-for-SGX-EPC-sections.patch

@@ -0,0 +1,277 @@
+From 1105812382e1126d86dddc16b3700f8c79dc93d1 Mon Sep 17 00:00:00 2001
+From: Yang Zhong <yang.zhong@intel.com>
+Date: Mon, 1 Nov 2021 12:20:05 -0400
+Subject: [PATCH 1/3] numa: Enable numa for SGX EPC sections
+
+The basic SGX did not enable numa for SGX EPC sections, which
+result in all EPC sections located in numa node 0. This patch
+enable SGX numa function in the guest and the EPC section can
+work with RAM as one numa node.
+
+The Guest kernel related log:
+[    0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
+[    0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
+The SRAT table can normally show SGX EPC sections menory info in different
+numa nodes.
+
+The SGX EPC numa related command:
+ ......
+ -m 4G,maxmem=20G \
+ -smp sockets=2,cores=2 \
+ -cpu host,+sgx-provisionkey \
+ -object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
+ -object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
+ -numa node,nodeid=0,cpus=0-1,memdev=node0 \
+ -object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
+ -object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
+ -numa node,nodeid=1,cpus=2-3,memdev=node1 \
+ -M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
+ ......
+
+Signed-off-by: Yang Zhong <yang.zhong@intel.com>
+Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
+Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
+---
+ hw/core/numa.c            |  5 ++---
+ hw/i386/acpi-build.c      |  2 ++
+ hw/i386/sgx-epc.c         |  3 +++
+ hw/i386/sgx-stub.c        |  4 ++++
+ hw/i386/sgx.c             | 44 +++++++++++++++++++++++++++++++++++++++
+ include/hw/i386/sgx-epc.h |  3 +++
+ monitor/hmp-cmds.c        |  1 +
+ qapi/machine.json         | 10 ++++++++-
+ qemu-options.hx           |  4 ++--
+ 9 files changed, 70 insertions(+), 6 deletions(-)
+
+diff --git a/hw/core/numa.c b/hw/core/numa.c
+index e6050b2273..1aa05dcf42 100644
+--- a/hw/core/numa.c
++++ b/hw/core/numa.c
+@@ -784,9 +784,8 @@ static void numa_stat_memory_devices(NumaNodeMem node_mem[])
+                 break;
+             case MEMORY_DEVICE_INFO_KIND_SGX_EPC:
+                 se = value->u.sgx_epc.data;
+-                /* TODO: once we support numa, assign to right node */
+-                node_mem[0].node_mem += se->size;
+-                node_mem[0].node_plugged_mem += se->size;
++                node_mem[se->node].node_mem += se->size;
++                node_mem[se->node].node_plugged_mem = 0;
+                 break;
+             default:
+                 g_assert_not_reached();
+diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
+index a99c6e4fe3..8383b83ee3 100644
+--- a/hw/i386/acpi-build.c
++++ b/hw/i386/acpi-build.c
+@@ -2068,6 +2068,8 @@ build_srat(GArray *table_data, BIOSLinker *linker, MachineState *machine)
+         nvdimm_build_srat(table_data);
+     }
+ 
++    sgx_epc_build_srat(table_data);
++
+     /*
+      * TODO: this part is not in ACPI spec and current linux kernel boots fine
+      * without these entries. But I recall there were issues the last time I
+diff --git a/hw/i386/sgx-epc.c b/hw/i386/sgx-epc.c
+index e508827e78..96b2940d75 100644
+--- a/hw/i386/sgx-epc.c
++++ b/hw/i386/sgx-epc.c
+@@ -21,6 +21,7 @@
+ 
+ static Property sgx_epc_properties[] = {
+     DEFINE_PROP_UINT64(SGX_EPC_ADDR_PROP, SGXEPCDevice, addr, 0),
++    DEFINE_PROP_UINT32(SGX_EPC_NUMA_NODE_PROP, SGXEPCDevice, node, 0),
+     DEFINE_PROP_LINK(SGX_EPC_MEMDEV_PROP, SGXEPCDevice, hostmem,
+                      TYPE_MEMORY_BACKEND_EPC, HostMemoryBackendEpc *),
+     DEFINE_PROP_END_OF_LIST(),
+@@ -139,6 +140,8 @@ static void sgx_epc_md_fill_device_info(const MemoryDeviceState *md,
+     se->memaddr = epc->addr;
+     se->size = object_property_get_uint(OBJECT(epc), SGX_EPC_SIZE_PROP,
+                                         NULL);
++    se->node = object_property_get_uint(OBJECT(epc), SGX_EPC_NUMA_NODE_PROP,
++                                        NULL);
+     se->memdev = object_get_canonical_path(OBJECT(epc->hostmem));
+ 
+     info->u.sgx_epc.data = se;
+diff --git a/hw/i386/sgx-stub.c b/hw/i386/sgx-stub.c
+index c9b379e665..26833eb233 100644
+--- a/hw/i386/sgx-stub.c
++++ b/hw/i386/sgx-stub.c
+@@ -6,6 +6,10 @@
+ #include "qapi/error.h"
+ #include "qapi/qapi-commands-misc-target.h"
+ 
++void sgx_epc_build_srat(GArray *table_data)
++{
++}
++
+ SGXInfo *qmp_query_sgx(Error **errp)
+ {
+     error_setg(errp, "SGX support is not compiled in");
+diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c
+index 8fef3dd8fa..d04299904a 100644
+--- a/hw/i386/sgx.c
++++ b/hw/i386/sgx.c
+@@ -23,6 +23,7 @@
+ #include "sysemu/hw_accel.h"
+ #include "sysemu/reset.h"
+ #include <sys/ioctl.h>
++#include "hw/acpi/aml-build.h"
+ 
+ #define SGX_MAX_EPC_SECTIONS            8
+ #define SGX_CPUID_EPC_INVALID           0x0
+@@ -36,6 +37,46 @@
+ 
+ #define RETRY_NUM                       2
+ 
++static int sgx_epc_device_list(Object *obj, void *opaque)
++{
++    GSList **list = opaque;
++
++    if (object_dynamic_cast(obj, TYPE_SGX_EPC)) {
++        *list = g_slist_append(*list, DEVICE(obj));
++    }
++
++    object_child_foreach(obj, sgx_epc_device_list, opaque);
++    return 0;
++}
++
++static GSList *sgx_epc_get_device_list(void)
++{
++    GSList *list = NULL;
++
++    object_child_foreach(qdev_get_machine(), sgx_epc_device_list, &list);
++    return list;
++}
++
++void sgx_epc_build_srat(GArray *table_data)
++{
++    GSList *device_list = sgx_epc_get_device_list();
++
++    for (; device_list; device_list = device_list->next) {
++        DeviceState *dev = device_list->data;
++        Object *obj = OBJECT(dev);
++        uint64_t addr, size;
++        int node;
++
++        node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
++                                        &error_abort);
++        addr = object_property_get_uint(obj, SGX_EPC_ADDR_PROP, &error_abort);
++        size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, &error_abort);
++
++        build_srat_memory(table_data, addr, size, node, MEM_AFFINITY_ENABLED);
++    }
++    g_slist_free(device_list);
++}
++
+ static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
+ {
+     return (low & MAKE_64BIT_MASK(12, 20)) +
+@@ -226,6 +267,9 @@ void pc_machine_init_sgx_epc(PCMachineState *pcms)
+         /* set the memdev link with memory backend */
+         object_property_parse(obj, SGX_EPC_MEMDEV_PROP, list->value->memdev,
+                               &error_fatal);
++        /* set the numa node property for sgx epc object */
++        object_property_set_uint(obj, SGX_EPC_NUMA_NODE_PROP, list->value->node,
++                             &error_fatal);
+         object_property_set_bool(obj, "realized", true, &error_fatal);
+         object_unref(obj);
+     }
+diff --git a/include/hw/i386/sgx-epc.h b/include/hw/i386/sgx-epc.h
+index a6a65be854..581fac389a 100644
+--- a/include/hw/i386/sgx-epc.h
++++ b/include/hw/i386/sgx-epc.h
+@@ -25,6 +25,7 @@
+ #define SGX_EPC_ADDR_PROP "addr"
+ #define SGX_EPC_SIZE_PROP "size"
+ #define SGX_EPC_MEMDEV_PROP "memdev"
++#define SGX_EPC_NUMA_NODE_PROP "node"
+ 
+ /**
+  * SGXEPCDevice:
+@@ -38,6 +39,7 @@ typedef struct SGXEPCDevice {
+ 
+     /* public */
+     uint64_t addr;
++    uint32_t node;
+     HostMemoryBackendEpc *hostmem;
+ } SGXEPCDevice;
+ 
+@@ -56,6 +58,7 @@ typedef struct SGXEPCState {
+ } SGXEPCState;
+ 
+ bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size);
++void sgx_epc_build_srat(GArray *table_data);
+ 
+ static inline uint64_t sgx_epc_above_4g_end(SGXEPCState *sgx_epc)
+ {
+diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
+index 9c91bf93e9..2669156b28 100644
+--- a/monitor/hmp-cmds.c
++++ b/monitor/hmp-cmds.c
+@@ -1810,6 +1810,7 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
+                                se->id ? se->id : "");
+                 monitor_printf(mon, "  memaddr: 0x%" PRIx64 "\n", se->memaddr);
+                 monitor_printf(mon, "  size: %" PRIu64 "\n", se->size);
++                monitor_printf(mon, "  node: %" PRId64 "\n", se->node);
+                 monitor_printf(mon, "  memdev: %s\n", se->memdev);
+                 break;
+             default:
+diff --git a/qapi/machine.json b/qapi/machine.json
+index f1839acf20..edeab6084b 100644
+--- a/qapi/machine.json
++++ b/qapi/machine.json
+@@ -1207,12 +1207,15 @@
+ #
+ # @memdev: memory backend linked with device
+ #
++# @node: the numa node
++#
+ # Since: 6.2
+ ##
+ { 'struct': 'SgxEPCDeviceInfo',
+   'data': { '*id': 'str',
+             'memaddr': 'size',
+             'size': 'size',
++            'node': 'int',
+             'memdev': 'str'
+           }
+ }
+@@ -1285,10 +1288,15 @@
+ #
+ # @memdev: memory backend linked with device
+ #
++# @node: the numa node
++#
+ # Since: 6.2
+ ##
+ { 'struct': 'SgxEPC',
+-  'data': { 'memdev': 'str' } }
++  'data': { 'memdev': 'str',
++            'node': 'int'
++          }
++}
+ 
+ ##
+ # @SgxEPCProperties:
+diff --git a/qemu-options.hx b/qemu-options.hx
+index ae2c6dbbfc..489b58e151 100644
+--- a/qemu-options.hx
++++ b/qemu-options.hx
+@@ -127,11 +127,11 @@ SRST
+ ERST
+ 
+ DEF("M", HAS_ARG, QEMU_OPTION_M,
+-    "                sgx-epc.0.memdev=memid\n",
++    "                sgx-epc.0.memdev=memid,sgx-epc.0.node=numaid\n",
+     QEMU_ARCH_ALL)
+ 
+ SRST
+-``sgx-epc.0.memdev=@var{memid}``
++``sgx-epc.0.memdev=@var{memid},sgx-epc.0.node=@var{numaid}``
+     Define an SGX EPC section.
+ ERST
+ 
+-- 
+2.25.1
+

tools/packaging/qemu/patches/6.2.x/0002-numa-Support-SGX-numa-in-the-monitor-and-Libvirt-int.patch

@@ -0,0 +1,200 @@
+From 4755927ae12547c2e7cb22c5fa1b39038c6c11b1 Mon Sep 17 00:00:00 2001
+From: Yang Zhong <yang.zhong@intel.com>
+Date: Mon, 1 Nov 2021 12:20:07 -0400
+Subject: [PATCH 2/3] numa: Support SGX numa in the monitor and Libvirt
+ interfaces
+
+Add the SGXEPCSection list into SGXInfo to show the multiple
+SGX EPC sections detailed info, not the total size like before.
+This patch can enable numa support for 'info sgx' command and
+QMP interfaces. The new interfaces show each EPC section info
+in one numa node. Libvirt can use QMP interface to get the
+detailed host SGX EPC capabilities to decide how to allocate
+host EPC sections to guest.
+
+(qemu) info sgx
+ SGX support: enabled
+ SGX1 support: enabled
+ SGX2 support: enabled
+ FLC support: enabled
+ NUMA node #0: size=67108864
+ NUMA node #1: size=29360128
+
+The QMP interface show:
+(QEMU) query-sgx
+{"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \
+[{"node": 0, "size": 67108864}, {"node": 1, "size": 29360128}], "flc": true}}
+
+(QEMU) query-sgx-capabilities
+{"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \
+[{"node": 0, "size": 17070817280}, {"node": 1, "size": 17079205888}], "flc": true}}
+
+Signed-off-by: Yang Zhong <yang.zhong@intel.com>
+Message-Id: <20211101162009.62161-4-yang.zhong@intel.com>
+Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
+---
+ hw/i386/sgx.c         | 51 +++++++++++++++++++++++++++++++++++--------
+ qapi/misc-target.json | 19 ++++++++++++++--
+ 2 files changed, 59 insertions(+), 11 deletions(-)
+
+diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c
+index d04299904a..5de5dd0893 100644
+--- a/hw/i386/sgx.c
++++ b/hw/i386/sgx.c
+@@ -83,11 +83,13 @@ static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
+            ((high & MAKE_64BIT_MASK(0, 20)) << 32);
+ }
+ 
+-static uint64_t sgx_calc_host_epc_section_size(void)
++static SGXEPCSectionList *sgx_calc_host_epc_sections(void)
+ {
++    SGXEPCSectionList *head = NULL, **tail = &head;
++    SGXEPCSection *section;
+     uint32_t i, type;
+     uint32_t eax, ebx, ecx, edx;
+-    uint64_t size = 0;
++    uint32_t j = 0;
+ 
+     for (i = 0; i < SGX_MAX_EPC_SECTIONS; i++) {
+         host_cpuid(0x12, i + 2, &eax, &ebx, &ecx, &edx);
+@@ -101,10 +103,13 @@ static uint64_t sgx_calc_host_epc_section_size(void)
+             break;
+         }
+ 
+-        size += sgx_calc_section_metric(ecx, edx);
++        section = g_new0(SGXEPCSection, 1);
++        section->node = j++;
++        section->size = sgx_calc_section_metric(ecx, edx);
++        QAPI_LIST_APPEND(tail, section);
+     }
+ 
+-    return size;
++    return head;
+ }
+ 
+ static void sgx_epc_reset(void *opaque)
+@@ -168,13 +173,35 @@ SGXInfo *qmp_query_sgx_capabilities(Error **errp)
+     info->sgx1 = eax & (1U << 0) ? true : false;
+     info->sgx2 = eax & (1U << 1) ? true : false;
+ 
+-    info->section_size = sgx_calc_host_epc_section_size();
++    info->sections = sgx_calc_host_epc_sections();
+ 
+     close(fd);
+ 
+     return info;
+ }
+ 
++static SGXEPCSectionList *sgx_get_epc_sections_list(void)
++{
++    GSList *device_list = sgx_epc_get_device_list();
++    SGXEPCSectionList *head = NULL, **tail = &head;
++    SGXEPCSection *section;
++
++    for (; device_list; device_list = device_list->next) {
++        DeviceState *dev = device_list->data;
++        Object *obj = OBJECT(dev);
++
++        section = g_new0(SGXEPCSection, 1);
++        section->node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
++                                                 &error_abort);
++        section->size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP,
++                                                 &error_abort);
++        QAPI_LIST_APPEND(tail, section);
++    }
++    g_slist_free(device_list);
++
++    return head;
++}
++
+ SGXInfo *qmp_query_sgx(Error **errp)
+ {
+     SGXInfo *info = NULL;
+@@ -193,14 +220,13 @@ SGXInfo *qmp_query_sgx(Error **errp)
+         return NULL;
+     }
+ 
+-    SGXEPCState *sgx_epc = &pcms->sgx_epc;
+     info = g_new0(SGXInfo, 1);
+ 
+     info->sgx = true;
+     info->sgx1 = true;
+     info->sgx2 = true;
+     info->flc = true;
+-    info->section_size = sgx_epc->size;
++    info->sections = sgx_get_epc_sections_list();
+ 
+     return info;
+ }
+@@ -208,6 +234,7 @@ SGXInfo *qmp_query_sgx(Error **errp)
+ void hmp_info_sgx(Monitor *mon, const QDict *qdict)
+ {
+     Error *err = NULL;
++    SGXEPCSectionList *section_list, *section;
+     g_autoptr(SGXInfo) info = qmp_query_sgx(&err);
+ 
+     if (err) {
+@@ -222,8 +249,14 @@ void hmp_info_sgx(Monitor *mon, const QDict *qdict)
+                    info->sgx2 ? "enabled" : "disabled");
+     monitor_printf(mon, "FLC support: %s\n",
+                    info->flc ? "enabled" : "disabled");
+-    monitor_printf(mon, "size: %" PRIu64 "\n",
+-                   info->section_size);
++
++    section_list = info->sections;
++    for (section = section_list; section; section = section->next) {
++        monitor_printf(mon, "NUMA node #%" PRId64 ": ",
++                       section->value->node);
++        monitor_printf(mon, "size=%" PRIu64 "\n",
++                       section->value->size);
++    }
+ }
+ 
+ bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size)
+diff --git a/qapi/misc-target.json b/qapi/misc-target.json
+index 5aa2b95b7d..1022aa0184 100644
+--- a/qapi/misc-target.json
++++ b/qapi/misc-target.json
+@@ -337,6 +337,21 @@
+   'if': 'TARGET_ARM' }
+ 
+ 
++##
++# @SGXEPCSection:
++#
++# Information about intel SGX EPC section info
++#
++# @node: the numa node
++#
++# @size: the size of epc section
++#
++# Since: 6.2
++##
++{ 'struct': 'SGXEPCSection',
++  'data': { 'node': 'int',
++            'size': 'uint64'}}
++
+ ##
+ # @SGXInfo:
+ #
+@@ -350,7 +365,7 @@
+ #
+ # @flc: true if FLC is supported
+ #
+-# @section-size: The EPC section size for guest
++# @sections: The EPC sections info for guest
+ #
+ # Since: 6.2
+ ##
+@@ -359,7 +374,7 @@
+             'sgx1': 'bool',
+             'sgx2': 'bool',
+             'flc': 'bool',
+-            'section-size': 'uint64'},
++            'sections': ['SGXEPCSection']},
+    'if': 'TARGET_I386' }
+ 
+ ##
+-- 
+2.25.1
+

tools/packaging/qemu/patches/6.2.x/0003-doc-Add-the-SGX-numa-description.patch

@@ -0,0 +1,67 @@
+From d1889b36098c79e2e6ac90faf3d0dc5ec0057677 Mon Sep 17 00:00:00 2001
+From: Yang Zhong <yang.zhong@intel.com>
+Date: Mon, 1 Nov 2021 12:20:08 -0400
+Subject: [PATCH 3/3] doc: Add the SGX numa description
+
+Add the SGX numa reference command and how to check if
+SGX numa is support or not with multiple EPC sections.
+
+Signed-off-by: Yang Zhong <yang.zhong@intel.com>
+Message-Id: <20211101162009.62161-5-yang.zhong@intel.com>
+Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
+---
+ docs/system/i386/sgx.rst | 31 +++++++++++++++++++++++++++----
+ 1 file changed, 27 insertions(+), 4 deletions(-)
+
+diff --git a/docs/system/i386/sgx.rst b/docs/system/i386/sgx.rst
+index f8fade5ac2..0f0a73f758 100644
+--- a/docs/system/i386/sgx.rst
++++ b/docs/system/i386/sgx.rst
+@@ -141,8 +141,7 @@ To launch a SGX guest:
+   |qemu_system_x86| \\
+    -cpu host,+sgx-provisionkey \\
+    -object memory-backend-epc,id=mem1,size=64M,prealloc=on \\
+-   -object memory-backend-epc,id=mem2,size=28M \\
+-   -M sgx-epc.0.memdev=mem1,sgx-epc.1.memdev=mem2
++   -M sgx-epc.0.memdev=mem1,sgx-epc.0.node=0
+ 
+ Utilizing SGX in the guest requires a kernel/OS with SGX support.
+ The support can be determined in guest by::
+@@ -152,8 +151,32 @@ The support can be determined in guest by::
+ and SGX epc info by::
+ 
+   $ dmesg | grep sgx
+-  [    1.242142] sgx: EPC section 0x180000000-0x181bfffff
+-  [    1.242319] sgx: EPC section 0x181c00000-0x1837fffff
++  [    0.182807] sgx: EPC section 0x140000000-0x143ffffff
++  [    0.183695] sgx: [Firmware Bug]: Unable to map EPC section to online node. Fallback to the NUMA node 0.
++
++To launch a SGX numa guest:
++
++.. parsed-literal::
++
++  |qemu_system_x86| \\
++   -cpu host,+sgx-provisionkey \\
++   -object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \\
++   -object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \\
++   -numa node,nodeid=0,cpus=0-1,memdev=node0 \\
++   -object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \\
++   -object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \\
++   -numa node,nodeid=1,cpus=2-3,memdev=node1 \\
++   -M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1
++
++and SGX epc numa info by::
++
++  $ dmesg | grep sgx
++  [    0.369937] sgx: EPC section 0x180000000-0x183ffffff
++  [    0.370259] sgx: EPC section 0x184000000-0x185bfffff
++
++  $ dmesg | grep SRAT
++  [    0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
++  [    0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
+ 
+ References
+ ----------
+-- 
+2.25.1
+

tools/packaging/scripts/configure-hypervisor.sh

@@ -322,7 +322,26 @@ generate_qemu_options() {
 	# But since QEMU 5.2 the daemon is built as part of the tools set
 	# (disabled with --disable-tools) thus it needs to be explicitely
 	# enabled.
-	qemu_options+=(functionality:--enable-virtiofsd)
+	#
+	# From Kata Containers 2.5.0-alpha2 x86_64 has been using the new
+	# implementation of virtiofs daemon, which is not part of QEMU.
+	# For the other arches, at least for now, keep building from while
+	# building QEMU.
+	#
+	# IOW, other arches are still using the C version of the virtiofsd.
+	case "$arch" in
+	aarch64)
+		qemu_options+=(functionality:--enable-virtiofsd)
+		;;
+	x86_64)
+		;;
+	ppc64le)
+		qemu_options+=(functionality:--enable-virtiofsd)
+		;;
+	s390x)
+		qemu_options+=(functionality:--enable-virtiofsd)
+		;;
+	esac
 	qemu_options+=(functionality:--enable-virtfs)
 
 	# Don't build linux-user bsd-user

tools/packaging/scripts/gen_versions_txt.sh

@@ -35,21 +35,21 @@ gen_version_file() {
 		ref="refs/tags/${kata_version}^{}"
 	fi
 
-	qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.version" "${kata_version}")
+	qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.version")
 	# Check if qemu.version can be used to get the version and hash, otherwise use qemu.tag
 	qemu_vanilla_ref="refs/heads/${qemu_vanilla_branch}"
 	if ! (git ls-remote --heads "https://github.com/qemu/qemu.git" | grep -q "refs/heads/${qemu_vanilla_branch}"); then
-		qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.tag" "${kata_version}")
+		qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.tag")
 		qemu_vanilla_ref="refs/tags/${qemu_vanilla_branch}^{}"
 	fi
 	qemu_vanilla_version=$(curl -s -L "https://raw.githubusercontent.com/qemu/qemu/${qemu_vanilla_branch}/VERSION")
 	qemu_vanilla_hash=$(git ls-remote https://github.com/qemu/qemu.git | grep "${qemu_vanilla_ref}" | awk '{print $1}')
 
-	kernel_version=$(get_from_kata_deps "assets.kernel.version" "${kata_version}")
+	kernel_version=$(get_from_kata_deps "assets.kernel.version")
 	#Remove extra 'v'
 	kernel_version=${kernel_version#v}
 
-	golang_version=$(get_from_kata_deps "languages.golang.meta.newest-version" "${kata_version}")
+	golang_version=$(get_from_kata_deps "languages.golang.meta.newest-version")
 
 	# - is not a valid char for rpmbuild
 	# see https://github.com/semver/semver/issues/145

tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh

@@ -14,7 +14,6 @@ ARCH=$(uname -m)
 [ "${ARCH}" != "aarch64" ] && [ "${ARCH}" != "x86_64" ] && exit
 
 script_dir=$(dirname $(readlink -f "$0"))
-kata_version="${kata_version:-}"
 force_build_from_source="${force_build_from_source:-false}"
 features="${features:-}"
 
@@ -27,7 +26,7 @@ cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
 
 if [ -z "$cloud_hypervisor_repo" ]; then
        info "Get cloud_hypervisor information from runtime versions.yaml"
-       cloud_hypervisor_url=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.url" "${kata_version}")
+       cloud_hypervisor_url=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.url")
        [ -n "$cloud_hypervisor_url" ] || die "failed to get cloud_hypervisor url"
        cloud_hypervisor_repo="${cloud_hypervisor_url}.git"
 fi
@@ -37,7 +36,7 @@ if [ -n "$cloud_hypervisor_pr" ]; then
 	force_build_from_source=true
 	cloud_hypervisor_version="PR $cloud_hypervisor_pr"
 else
-	[ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version" "${kata_version}")
+	[ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")
 	[ -n "$cloud_hypervisor_version" ] || die "failed to get cloud_hypervisor version"
 fi
 

tools/packaging/static-build/firecracker/build-static-firecracker.sh

@@ -17,17 +17,16 @@ config_dir="${script_dir}/../../scripts/"
 firecracker_repo="${firecracker_repo:-}"
 firecracker_dir="firecracker"
 firecracker_version="${firecracker_version:-}"
-kata_version="${kata_version:-}"
 
 if [ -z "$firecracker_repo" ]; then
 	info "Get firecracker information from runtime versions.yaml"
-        firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url" "${kata_version}")
+        firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url")
 	[ -n "$firecracker_url" ] || die "failed to get firecracker url"
         firecracker_repo="${firecracker_url}.git"
 fi
 [ -n "$firecracker_repo" ] || die "failed to get firecracker repo"
 
-[ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version" "${kata_version}")
+[ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version")
 [ -n "$firecracker_version" ] || die "failed to get firecracker version"
 
 info "Build ${firecracker_repo} version: ${firecracker_version}"

tools/packaging/static-build/scripts/qemu-build-post.sh

@@ -26,7 +26,9 @@ done
 if [[ -n "${BUILD_SUFFIX}" ]]; then
 	echo "Rename binaries using $BUILD_SUFFIX"
 	find -name 'qemu-system-*' -exec mv {} {}-experimental \;
-	find -name 'virtiofsd' -exec mv {} {}-experimental \;
+	if [[ ${ARCH} != "x86_64" ]]; then
+		find -name 'virtiofsd' -exec mv {} {}-experimental \;
+	fi
 fi
 
 echo "INFO: create the tarball"

tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ARCH=$(uname -m)
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${script_dir}/../../scripts/lib.sh"
+
+virtiofsd_version="${virtiofsd_version:-}"
+
+[ -n "$virtiofsd_version" ] || virtiofsd_version=$(get_from_kata_deps "externals.virtiofsd.version")
+[ -n "$virtiofsd_version" ] || die "failed to get virtiofsd version"
+
+if [ "${ARCH}" != "x86_64" ]; then
+	info "Only x86_64 binaries are distributed as part of the virtiofsd releases" && exit 1
+fi
+
+pull_virtiofsd_released_binary() {
+    info "Download virtiofsd version: ${virtiofsd_version}"
+    virtiofsd_zip=$(get_from_kata_deps "externals.virtiofsd.meta.binary")
+    [ -n "${virtiofsd_zip}" ] || die "failed to get virtiofsd binary URL"
+
+    mkdir -p virtiofsd
+
+    pushd virtiofsd
+    curl --fail -L ${virtiofsd_zip} -o virtiofsd.zip || return 1
+    unzip virtiofsd.zip
+    mv -f target/x86_64-unknown-linux-musl/release/virtiofsd virtiofsd
+    chmod +x virtiofsd
+    rm -rf target
+    rm virtiofsd.zip
+    popd
+}
+
+pull_virtiofsd_released_binary

versions.yaml

@@ -256,6 +256,17 @@ externals:
     url: "https://github.com/containerd/nydus-snapshotter"
     version: "v0.1.0"
 
+  virtiofsd:
+    description: "vhost-user virtio-fs device backend written in Rust"
+    url: "https://gitlab.com/virtio-fs/virtiofsd"
+    version: "v1.2.0"
+    meta:
+      # From https://gitlab.com/virtio-fs/virtiofsd/-/releases/v1.2.0,
+      # this is the link labelled virtiofsd-v1.2.0.zip
+      #
+      # yamllint disable-line rule:line-length
+      binary: "https://gitlab.com/virtio-fs/virtiofsd/uploads/b26d9891caac83209a3fdd24bcf2ae86/virtiofsd-v1.2.0.zip"
+
 languages:
   description: |
     Details of programming languages required to build system