runtime: Dynamically discover hugepage size

buildNUMATopologyForVFIOHostSet assumed a fixed 2 MiB huge page size
(memAlign = 2) when distributing memory across guest NUMA nodes. On
hosts where /dev/hugepages is backed by 1 GiB (or other) huge pages,
per-node memory was not aligned to the actual page size, so QEMU could
not back the memory with huge pages unless the operator intentionally
aligned memory requests to 1GiB.

Inspect the real huge page size via statfs on /dev/hugepages, which
reports the huge page size as the hugetlbfs filesystem block size, and
align per-node memory to that value instead of assuming 2 MiB.

Update TestBuildNUMATopologyHugePages to derive the expected per-node
memory from the detected page size so it holds on both 2 MiB and 1 GiB
hosts.

Signed-off-by: LandonTClipp <lclipp@coreweave.com>
This commit is contained in:
LandonTClipp
2026-07-10 18:14:56 +00:00
parent 7e48711ec6
commit 71aecf91e1
2 changed files with 51 additions and 4 deletions

View File

@@ -628,6 +628,31 @@ func (q *qemu) buildNUMATopology() ([]govmmQemu.NUMANode, []govmmQemu.NUMADist,
return q.buildNUMATopologyForVFIOHostSet(vfioHostSet)
}
// defaultHugepagesMountpoint is the standard mount point of the hugetlbfs
// filesystem used to back guest memory with huge pages.
const defaultHugepagesMountpoint = "/dev/hugepages"
// hugepageSizeBytes returns the size, in bytes, of a single huge page backing
// the hugetlbfs mounted at path. hugetlbfs reports the huge page size as its
// filesystem block size, so we can discover the actual page size (e.g. 2 MiB
// or 1 GiB) via statfs without assuming a fixed value.
func hugepageSizeBytes(path string) (uint64, error) {
var fs unix.Statfs_t
if err := unix.Statfs(path, &fs); err != nil {
return 0, fmt.Errorf("statfs %q: %w", path, err)
}
if fs.Bsize <= 0 {
return 0, fmt.Errorf("hugepage mount %q reported invalid block size %d", path, fs.Bsize)
}
hpSize := uint64(fs.Bsize)
mib := uint64(1) << utils.MibToBytesShift
if hpSize < mib || hpSize%mib != 0 {
return 0, fmt.Errorf("hugepage mount %q reported unexpected block size %d", path, fs.Bsize)
}
return hpSize, nil
}
func (q *qemu) buildNUMATopologyForVFIOHostSet(vfioHostSet map[int]struct{}) ([]govmmQemu.NUMANode, []govmmQemu.NUMADist, error) {
// q.config.GuestNUMANodes has already been right-sized (when applicable)
// by maybeRightSizeAutoNUMA() at hypervisor setup time. Empty means
@@ -703,14 +728,26 @@ func (q *qemu) buildNUMATopologyForVFIOHostSet(vfioHostSet map[int]struct{}) ([]
var memAlign uint64 = 1
if q.config.HugePages {
memAlign = 2
// Align per-node memory to the actual huge page size backing
// /dev/hugepages rather than assuming 2 MiB. Hosts may configure a
// default huge page size of 1 GiB (or other), in which case each
// node's memory must be a multiple of that size for QEMU to back it
// with huge pages.
hpSize, err := hugepageSizeBytes(defaultHugepagesMountpoint)
if err != nil {
return nil, nil, fmt.Errorf("failed to determine huge page size for NUMA memory alignment: %w", err)
}
memAlign = hpSize >> utils.MibToBytesShift
if memAlign == 0 {
memAlign = 1
}
}
backendType := "memory-backend-ram"
backendPath := ""
if q.config.HugePages {
backendType = "memory-backend-file"
backendPath = "/dev/hugepages"
backendPath = defaultHugepagesMountpoint
} else if q.config.SharedFS == config.VirtioFS || q.config.SharedFS == config.VirtioFSNydus {
backendType = "memory-backend-file"
backendPath = fallbackFileBackedMemDir

View File

@@ -1350,10 +1350,20 @@ func TestBuildNUMATopologyHugePages(t *testing.T) {
t.Skip("skipping: /dev/hugepages not available")
}
assert := assert.New(t)
// Derive the expected per-node memory from the actual huge page size
// backing /dev/hugepages, so the test holds regardless of whether the
// host uses 2 MiB or 1 GiB huge pages.
hpBytes, err := hugepageSizeBytes(defaultHugepagesMountpoint)
assert.NoError(err)
alignMiB := hpBytes >> utils.MibToBytesShift
assert.NotZero(alignMiB)
// Give each of the two nodes two huge pages worth of memory so the total
// is evenly divisible and huge-page aligned on any host.
perNodeMiB := alignMiB * 2
q := &qemu{
config: HypervisorConfig{
DefaultMaxVCPUs: 4,
MemorySize: 1024,
MemorySize: uint32(perNodeMiB * 2),
HugePages: true,
GuestNUMANodes: []types.GuestNUMANode{
{HostNodes: "0", HostCPUs: "0-1"},
@@ -1366,7 +1376,7 @@ func TestBuildNUMATopologyHugePages(t *testing.T) {
assert.Len(nodes, 2)
assert.Equal("memory-backend-file", nodes[0].MemBackendType)
assert.Equal("/dev/hugepages", nodes[0].MemBackendPath)
assert.Equal("512M", nodes[0].MemSize)
assert.Equal(fmt.Sprintf("%dM", perNodeMiB), nodes[0].MemSize)
}
func TestBuildNUMATopologyVirtioFS(t *testing.T) {