github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-15 08:02:31 +00:00
Code Issues Projects Releases Wiki Activity

tools: Add QEMU patches for SGX numa support

Browse Source 
There are a few patches for SGX numa support in QEMU added after the
6.2.0 release. Add them for SGX support in Kata.

Fixes #4254

Signed-off-by: Chelsea Mafrica <chelsea.e.mafrica@intel.com>
(cherry picked from commit b4b9068cb7)
This commit is contained in:
Chelsea Mafrica 2022-05-13 16:28:13 -07:00
parent b93a0b1012
commit 3413c8588d
3 changed files with 544 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
tools/packaging/qemu/patches/6.2.x/0001-numa-Enable-numa-for-SGX-EPC-sections.patch Normal file
View File

@ -0,0 +1,277 @@
From 1105812382e1126d86dddc16b3700f8c79dc93d1 Mon Sep 17 00:00:00 2001
From: Yang Zhong <yang.zhong@intel.com>
Date: Mon, 1 Nov 2021 12:20:05 -0400
Subject: [PATCH 1/3] numa: Enable numa for SGX EPC sections
The basic SGX did not enable numa for SGX EPC sections, which
result in all EPC sections located in numa node 0. This patch
enable SGX numa function in the guest and the EPC section can
work with RAM as one numa node.
The Guest kernel related log:
[ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
[ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
The SRAT table can normally show SGX EPC sections menory info in different
numa nodes.
The SGX EPC numa related command:
......
-m 4G,maxmem=20G \
-smp sockets=2,cores=2 \
-cpu host,+sgx-provisionkey \
-object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \
-object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \
-numa node,nodeid=0,cpus=0-1,memdev=node0 \
-object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \
-object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \
-numa node,nodeid=1,cpus=2-3,memdev=node1 \
-M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1 \
......
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-2-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
hw/core/numa.c | 5 ++---
hw/i386/acpi-build.c | 2 ++
hw/i386/sgx-epc.c | 3 +++
hw/i386/sgx-stub.c | 4 ++++
hw/i386/sgx.c | 44 +++++++++++++++++++++++++++++++++++++++
include/hw/i386/sgx-epc.h | 3 +++
monitor/hmp-cmds.c | 1 +
qapi/machine.json | 10 ++++++++-
qemu-options.hx | 4 ++--
9 files changed, 70 insertions(+), 6 deletions(-)
diff --git a/hw/core/numa.c b/hw/core/numa.c
index e6050b2273..1aa05dcf42 100644
--- a/hw/core/numa.c
+++ b/hw/core/numa.c
@@ -784,9 +784,8 @@ static void numa_stat_memory_devices(NumaNodeMem node_mem[])
break;
case MEMORY_DEVICE_INFO_KIND_SGX_EPC:
se = value->u.sgx_epc.data;
- /* TODO: once we support numa, assign to right node */
- node_mem[0].node_mem += se->size;
- node_mem[0].node_plugged_mem += se->size;
+ node_mem[se->node].node_mem += se->size;
+ node_mem[se->node].node_plugged_mem = 0;
break;
default:
g_assert_not_reached();
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index a99c6e4fe3..8383b83ee3 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -2068,6 +2068,8 @@ build_srat(GArray *table_data, BIOSLinker *linker, MachineState *machine)
nvdimm_build_srat(table_data);
}
+ sgx_epc_build_srat(table_data);
+
/*
* TODO: this part is not in ACPI spec and current linux kernel boots fine
* without these entries. But I recall there were issues the last time I
diff --git a/hw/i386/sgx-epc.c b/hw/i386/sgx-epc.c
index e508827e78..96b2940d75 100644
--- a/hw/i386/sgx-epc.c
+++ b/hw/i386/sgx-epc.c
@@ -21,6 +21,7 @@
static Property sgx_epc_properties[] = {
DEFINE_PROP_UINT64(SGX_EPC_ADDR_PROP, SGXEPCDevice, addr, 0),
+ DEFINE_PROP_UINT32(SGX_EPC_NUMA_NODE_PROP, SGXEPCDevice, node, 0),
DEFINE_PROP_LINK(SGX_EPC_MEMDEV_PROP, SGXEPCDevice, hostmem,
TYPE_MEMORY_BACKEND_EPC, HostMemoryBackendEpc *),
DEFINE_PROP_END_OF_LIST(),
@@ -139,6 +140,8 @@ static void sgx_epc_md_fill_device_info(const MemoryDeviceState *md,
se->memaddr = epc->addr;
se->size = object_property_get_uint(OBJECT(epc), SGX_EPC_SIZE_PROP,
NULL);
+ se->node = object_property_get_uint(OBJECT(epc), SGX_EPC_NUMA_NODE_PROP,
+ NULL);
se->memdev = object_get_canonical_path(OBJECT(epc->hostmem));
info->u.sgx_epc.data = se;
diff --git a/hw/i386/sgx-stub.c b/hw/i386/sgx-stub.c
index c9b379e665..26833eb233 100644
--- a/hw/i386/sgx-stub.c
+++ b/hw/i386/sgx-stub.c
@@ -6,6 +6,10 @@
#include "qapi/error.h"
#include "qapi/qapi-commands-misc-target.h"
+void sgx_epc_build_srat(GArray *table_data)
+{
+}
+
SGXInfo *qmp_query_sgx(Error **errp)
{
error_setg(errp, "SGX support is not compiled in");
diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c
index 8fef3dd8fa..d04299904a 100644
--- a/hw/i386/sgx.c
+++ b/hw/i386/sgx.c
@@ -23,6 +23,7 @@
#include "sysemu/hw_accel.h"
#include "sysemu/reset.h"
#include <sys/ioctl.h>
+#include "hw/acpi/aml-build.h"
#define SGX_MAX_EPC_SECTIONS 8
#define SGX_CPUID_EPC_INVALID 0x0
@@ -36,6 +37,46 @@
#define RETRY_NUM 2
+static int sgx_epc_device_list(Object *obj, void *opaque)
+{
+ GSList **list = opaque;
+
+ if (object_dynamic_cast(obj, TYPE_SGX_EPC)) {
+ *list = g_slist_append(*list, DEVICE(obj));
+ }
+
+ object_child_foreach(obj, sgx_epc_device_list, opaque);
+ return 0;
+}
+
+static GSList *sgx_epc_get_device_list(void)
+{
+ GSList *list = NULL;
+
+ object_child_foreach(qdev_get_machine(), sgx_epc_device_list, &list);
+ return list;
+}
+
+void sgx_epc_build_srat(GArray *table_data)
+{
+ GSList *device_list = sgx_epc_get_device_list();
+
+ for (; device_list; device_list = device_list->next) {
+ DeviceState *dev = device_list->data;
+ Object *obj = OBJECT(dev);
+ uint64_t addr, size;
+ int node;
+
+ node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
+ &error_abort);
+ addr = object_property_get_uint(obj, SGX_EPC_ADDR_PROP, &error_abort);
+ size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP, &error_abort);
+
+ build_srat_memory(table_data, addr, size, node, MEM_AFFINITY_ENABLED);
+ }
+ g_slist_free(device_list);
+}
+
static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
{
return (low & MAKE_64BIT_MASK(12, 20)) +
@@ -226,6 +267,9 @@ void pc_machine_init_sgx_epc(PCMachineState *pcms)
/* set the memdev link with memory backend */
object_property_parse(obj, SGX_EPC_MEMDEV_PROP, list->value->memdev,
&error_fatal);
+ /* set the numa node property for sgx epc object */
+ object_property_set_uint(obj, SGX_EPC_NUMA_NODE_PROP, list->value->node,
+ &error_fatal);
object_property_set_bool(obj, "realized", true, &error_fatal);
object_unref(obj);
}
diff --git a/include/hw/i386/sgx-epc.h b/include/hw/i386/sgx-epc.h
index a6a65be854..581fac389a 100644
--- a/include/hw/i386/sgx-epc.h
+++ b/include/hw/i386/sgx-epc.h
@@ -25,6 +25,7 @@
#define SGX_EPC_ADDR_PROP "addr"
#define SGX_EPC_SIZE_PROP "size"
#define SGX_EPC_MEMDEV_PROP "memdev"
+#define SGX_EPC_NUMA_NODE_PROP "node"
/**
* SGXEPCDevice:
@@ -38,6 +39,7 @@ typedef struct SGXEPCDevice {
/* public */
uint64_t addr;
+ uint32_t node;
HostMemoryBackendEpc *hostmem;
} SGXEPCDevice;
@@ -56,6 +58,7 @@ typedef struct SGXEPCState {
} SGXEPCState;
bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size);
+void sgx_epc_build_srat(GArray *table_data);
static inline uint64_t sgx_epc_above_4g_end(SGXEPCState *sgx_epc)
{
diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
index 9c91bf93e9..2669156b28 100644
--- a/monitor/hmp-cmds.c
+++ b/monitor/hmp-cmds.c
@@ -1810,6 +1810,7 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
se->id ? se->id : "");
monitor_printf(mon, " memaddr: 0x%" PRIx64 "\n", se->memaddr);
monitor_printf(mon, " size: %" PRIu64 "\n", se->size);
+ monitor_printf(mon, " node: %" PRId64 "\n", se->node);
monitor_printf(mon, " memdev: %s\n", se->memdev);
break;
default:
diff --git a/qapi/machine.json b/qapi/machine.json
index f1839acf20..edeab6084b 100644
--- a/qapi/machine.json
+++ b/qapi/machine.json
@@ -1207,12 +1207,15 @@
#
# @memdev: memory backend linked with device
#
+# @node: the numa node
+#
# Since: 6.2
##
{ 'struct': 'SgxEPCDeviceInfo',
'data': { '*id': 'str',
'memaddr': 'size',
'size': 'size',
+ 'node': 'int',
'memdev': 'str'
}
}
@@ -1285,10 +1288,15 @@
#
# @memdev: memory backend linked with device
#
+# @node: the numa node
+#
# Since: 6.2
##
{ 'struct': 'SgxEPC',
- 'data': { 'memdev': 'str' } }
+ 'data': { 'memdev': 'str',
+ 'node': 'int'
+ }
+}
##
# @SgxEPCProperties:
diff --git a/qemu-options.hx b/qemu-options.hx
index ae2c6dbbfc..489b58e151 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -127,11 +127,11 @@ SRST
ERST
DEF("M", HAS_ARG, QEMU_OPTION_M,
- " sgx-epc.0.memdev=memid\n",
+ " sgx-epc.0.memdev=memid,sgx-epc.0.node=numaid\n",
QEMU_ARCH_ALL)
SRST
-``sgx-epc.0.memdev=@var{memid}``
+``sgx-epc.0.memdev=@var{memid},sgx-epc.0.node=@var{numaid}``
Define an SGX EPC section.
ERST
--
2.25.1

200
tools/packaging/qemu/patches/6.2.x/0002-numa-Support-SGX-numa-in-the-monitor-and-Libvirt-int.patch Normal file
View File

@ -0,0 +1,200 @@
From 4755927ae12547c2e7cb22c5fa1b39038c6c11b1 Mon Sep 17 00:00:00 2001
From: Yang Zhong <yang.zhong@intel.com>
Date: Mon, 1 Nov 2021 12:20:07 -0400
Subject: [PATCH 2/3] numa: Support SGX numa in the monitor and Libvirt
interfaces
Add the SGXEPCSection list into SGXInfo to show the multiple
SGX EPC sections detailed info, not the total size like before.
This patch can enable numa support for 'info sgx' command and
QMP interfaces. The new interfaces show each EPC section info
in one numa node. Libvirt can use QMP interface to get the
detailed host SGX EPC capabilities to decide how to allocate
host EPC sections to guest.
(qemu) info sgx
SGX support: enabled
SGX1 support: enabled
SGX2 support: enabled
FLC support: enabled
NUMA node #0: size=67108864
NUMA node #1: size=29360128
The QMP interface show:
(QEMU) query-sgx
{"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \
[{"node": 0, "size": 67108864}, {"node": 1, "size": 29360128}], "flc": true}}
(QEMU) query-sgx-capabilities
{"return": {"sgx": true, "sgx2": true, "sgx1": true, "sections": \
[{"node": 0, "size": 17070817280}, {"node": 1, "size": 17079205888}], "flc": true}}
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-4-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
hw/i386/sgx.c | 51 +++++++++++++++++++++++++++++++++++--------
qapi/misc-target.json | 19 ++++++++++++++--
2 files changed, 59 insertions(+), 11 deletions(-)
diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c
index d04299904a..5de5dd0893 100644
--- a/hw/i386/sgx.c
+++ b/hw/i386/sgx.c
@@ -83,11 +83,13 @@ static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high)
((high & MAKE_64BIT_MASK(0, 20)) << 32);
}
-static uint64_t sgx_calc_host_epc_section_size(void)
+static SGXEPCSectionList *sgx_calc_host_epc_sections(void)
{
+ SGXEPCSectionList *head = NULL, **tail = &head;
+ SGXEPCSection *section;
uint32_t i, type;
uint32_t eax, ebx, ecx, edx;
- uint64_t size = 0;
+ uint32_t j = 0;
for (i = 0; i < SGX_MAX_EPC_SECTIONS; i++) {
host_cpuid(0x12, i + 2, &eax, &ebx, &ecx, &edx);
@@ -101,10 +103,13 @@ static uint64_t sgx_calc_host_epc_section_size(void)
break;
}
- size += sgx_calc_section_metric(ecx, edx);
+ section = g_new0(SGXEPCSection, 1);
+ section->node = j++;
+ section->size = sgx_calc_section_metric(ecx, edx);
+ QAPI_LIST_APPEND(tail, section);
}
- return size;
+ return head;
}
static void sgx_epc_reset(void *opaque)
@@ -168,13 +173,35 @@ SGXInfo *qmp_query_sgx_capabilities(Error **errp)
info->sgx1 = eax & (1U << 0) ? true : false;
info->sgx2 = eax & (1U << 1) ? true : false;
- info->section_size = sgx_calc_host_epc_section_size();
+ info->sections = sgx_calc_host_epc_sections();
close(fd);
return info;
}
+static SGXEPCSectionList *sgx_get_epc_sections_list(void)
+{
+ GSList *device_list = sgx_epc_get_device_list();
+ SGXEPCSectionList *head = NULL, **tail = &head;
+ SGXEPCSection *section;
+
+ for (; device_list; device_list = device_list->next) {
+ DeviceState *dev = device_list->data;
+ Object *obj = OBJECT(dev);
+
+ section = g_new0(SGXEPCSection, 1);
+ section->node = object_property_get_uint(obj, SGX_EPC_NUMA_NODE_PROP,
+ &error_abort);
+ section->size = object_property_get_uint(obj, SGX_EPC_SIZE_PROP,
+ &error_abort);
+ QAPI_LIST_APPEND(tail, section);
+ }
+ g_slist_free(device_list);
+
+ return head;
+}
+
SGXInfo *qmp_query_sgx(Error **errp)
{
SGXInfo *info = NULL;
@@ -193,14 +220,13 @@ SGXInfo *qmp_query_sgx(Error **errp)
return NULL;
}
- SGXEPCState *sgx_epc = &pcms->sgx_epc;
info = g_new0(SGXInfo, 1);
info->sgx = true;
info->sgx1 = true;
info->sgx2 = true;
info->flc = true;
- info->section_size = sgx_epc->size;
+ info->sections = sgx_get_epc_sections_list();
return info;
}
@@ -208,6 +234,7 @@ SGXInfo *qmp_query_sgx(Error **errp)
void hmp_info_sgx(Monitor *mon, const QDict *qdict)
{
Error *err = NULL;
+ SGXEPCSectionList *section_list, *section;
g_autoptr(SGXInfo) info = qmp_query_sgx(&err);
if (err) {
@@ -222,8 +249,14 @@ void hmp_info_sgx(Monitor *mon, const QDict *qdict)
info->sgx2 ? "enabled" : "disabled");
monitor_printf(mon, "FLC support: %s\n",
info->flc ? "enabled" : "disabled");
- monitor_printf(mon, "size: %" PRIu64 "\n",
- info->section_size);
+
+ section_list = info->sections;
+ for (section = section_list; section; section = section->next) {
+ monitor_printf(mon, "NUMA node #%" PRId64 ": ",
+ section->value->node);
+ monitor_printf(mon, "size=%" PRIu64 "\n",
+ section->value->size);
+ }
}
bool sgx_epc_get_section(int section_nr, uint64_t *addr, uint64_t *size)
diff --git a/qapi/misc-target.json b/qapi/misc-target.json
index 5aa2b95b7d..1022aa0184 100644
--- a/qapi/misc-target.json
+++ b/qapi/misc-target.json
@@ -337,6 +337,21 @@
'if': 'TARGET_ARM' }
+##
+# @SGXEPCSection:
+#
+# Information about intel SGX EPC section info
+#
+# @node: the numa node
+#
+# @size: the size of epc section
+#
+# Since: 6.2
+##
+{ 'struct': 'SGXEPCSection',
+ 'data': { 'node': 'int',
+ 'size': 'uint64'}}
+
##
# @SGXInfo:
#
@@ -350,7 +365,7 @@
#
# @flc: true if FLC is supported
#
-# @section-size: The EPC section size for guest
+# @sections: The EPC sections info for guest
#
# Since: 6.2
##
@@ -359,7 +374,7 @@
'sgx1': 'bool',
'sgx2': 'bool',
'flc': 'bool',
- 'section-size': 'uint64'},
+ 'sections': ['SGXEPCSection']},
'if': 'TARGET_I386' }
##
--
2.25.1

67
tools/packaging/qemu/patches/6.2.x/0003-doc-Add-the-SGX-numa-description.patch Normal file
View File

@ -0,0 +1,67 @@
From d1889b36098c79e2e6ac90faf3d0dc5ec0057677 Mon Sep 17 00:00:00 2001
From: Yang Zhong <yang.zhong@intel.com>
Date: Mon, 1 Nov 2021 12:20:08 -0400
Subject: [PATCH 3/3] doc: Add the SGX numa description
Add the SGX numa reference command and how to check if
SGX numa is support or not with multiple EPC sections.
Signed-off-by: Yang Zhong <yang.zhong@intel.com>
Message-Id: <20211101162009.62161-5-yang.zhong@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
docs/system/i386/sgx.rst | 31 +++++++++++++++++++++++++++----
1 file changed, 27 insertions(+), 4 deletions(-)
diff --git a/docs/system/i386/sgx.rst b/docs/system/i386/sgx.rst
index f8fade5ac2..0f0a73f758 100644
--- a/docs/system/i386/sgx.rst
+++ b/docs/system/i386/sgx.rst
@@ -141,8 +141,7 @@ To launch a SGX guest:
|qemu_system_x86| \\
-cpu host,+sgx-provisionkey \\
-object memory-backend-epc,id=mem1,size=64M,prealloc=on \\
- -object memory-backend-epc,id=mem2,size=28M \\
- -M sgx-epc.0.memdev=mem1,sgx-epc.1.memdev=mem2
+ -M sgx-epc.0.memdev=mem1,sgx-epc.0.node=0
Utilizing SGX in the guest requires a kernel/OS with SGX support.
The support can be determined in guest by::
@@ -152,8 +151,32 @@ The support can be determined in guest by::
and SGX epc info by::
$ dmesg | grep sgx
- [ 1.242142] sgx: EPC section 0x180000000-0x181bfffff
- [ 1.242319] sgx: EPC section 0x181c00000-0x1837fffff
+ [ 0.182807] sgx: EPC section 0x140000000-0x143ffffff
+ [ 0.183695] sgx: [Firmware Bug]: Unable to map EPC section to online node. Fallback to the NUMA node 0.
+
+To launch a SGX numa guest:
+
+.. parsed-literal::
+
+ |qemu_system_x86| \\
+ -cpu host,+sgx-provisionkey \\
+ -object memory-backend-ram,size=2G,host-nodes=0,policy=bind,id=node0 \\
+ -object memory-backend-epc,id=mem0,size=64M,prealloc=on,host-nodes=0,policy=bind \\
+ -numa node,nodeid=0,cpus=0-1,memdev=node0 \\
+ -object memory-backend-ram,size=2G,host-nodes=1,policy=bind,id=node1 \\
+ -object memory-backend-epc,id=mem1,size=28M,prealloc=on,host-nodes=1,policy=bind \\
+ -numa node,nodeid=1,cpus=2-3,memdev=node1 \\
+ -M sgx-epc.0.memdev=mem0,sgx-epc.0.node=0,sgx-epc.1.memdev=mem1,sgx-epc.1.node=1
+
+and SGX epc numa info by::
+
+ $ dmesg | grep sgx
+ [ 0.369937] sgx: EPC section 0x180000000-0x183ffffff
+ [ 0.370259] sgx: EPC section 0x184000000-0x185bfffff
+
+ $ dmesg | grep SRAT
+ [ 0.009981] ACPI: SRAT: Node 0 PXM 0 [mem 0x180000000-0x183ffffff]
+ [ 0.009982] ACPI: SRAT: Node 1 PXM 1 [mem 0x184000000-0x185bfffff]
References
----------
--
2.25.1