diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a94bf4845b..b21ef94bba 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -628,6 +628,31 @@ func (q *qemu) buildNUMATopology() ([]govmmQemu.NUMANode, []govmmQemu.NUMADist, return q.buildNUMATopologyForVFIOHostSet(vfioHostSet) } +// defaultHugepagesMountpoint is the standard mount point of the hugetlbfs +// filesystem used to back guest memory with huge pages. +const defaultHugepagesMountpoint = "/dev/hugepages" + +// hugepageSizeBytes returns the size, in bytes, of a single huge page backing +// the hugetlbfs mounted at path. hugetlbfs reports the huge page size as its +// filesystem block size, so we can discover the actual page size (e.g. 2 MiB +// or 1 GiB) via statfs without assuming a fixed value. +func hugepageSizeBytes(path string) (uint64, error) { + var fs unix.Statfs_t + if err := unix.Statfs(path, &fs); err != nil { + return 0, fmt.Errorf("statfs %q: %w", path, err) + } + if fs.Bsize <= 0 { + return 0, fmt.Errorf("hugepage mount %q reported invalid block size %d", path, fs.Bsize) + } + + hpSize := uint64(fs.Bsize) + mib := uint64(1) << utils.MibToBytesShift + if hpSize < mib || hpSize%mib != 0 { + return 0, fmt.Errorf("hugepage mount %q reported unexpected block size %d", path, fs.Bsize) + } + return hpSize, nil +} + func (q *qemu) buildNUMATopologyForVFIOHostSet(vfioHostSet map[int]struct{}) ([]govmmQemu.NUMANode, []govmmQemu.NUMADist, error) { // q.config.GuestNUMANodes has already been right-sized (when applicable) // by maybeRightSizeAutoNUMA() at hypervisor setup time. Empty means @@ -703,14 +728,26 @@ func (q *qemu) buildNUMATopologyForVFIOHostSet(vfioHostSet map[int]struct{}) ([] var memAlign uint64 = 1 if q.config.HugePages { - memAlign = 2 + // Align per-node memory to the actual huge page size backing + // /dev/hugepages rather than assuming 2 MiB. Hosts may configure a + // default huge page size of 1 GiB (or other), in which case each + // node's memory must be a multiple of that size for QEMU to back it + // with huge pages. + hpSize, err := hugepageSizeBytes(defaultHugepagesMountpoint) + if err != nil { + return nil, nil, fmt.Errorf("failed to determine huge page size for NUMA memory alignment: %w", err) + } + memAlign = hpSize >> utils.MibToBytesShift + if memAlign == 0 { + memAlign = 1 + } } backendType := "memory-backend-ram" backendPath := "" if q.config.HugePages { backendType = "memory-backend-file" - backendPath = "/dev/hugepages" + backendPath = defaultHugepagesMountpoint } else if q.config.SharedFS == config.VirtioFS || q.config.SharedFS == config.VirtioFSNydus { backendType = "memory-backend-file" backendPath = fallbackFileBackedMemDir diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index 61248dca13..efc6e94d29 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -1350,10 +1350,20 @@ func TestBuildNUMATopologyHugePages(t *testing.T) { t.Skip("skipping: /dev/hugepages not available") } assert := assert.New(t) + // Derive the expected per-node memory from the actual huge page size + // backing /dev/hugepages, so the test holds regardless of whether the + // host uses 2 MiB or 1 GiB huge pages. + hpBytes, err := hugepageSizeBytes(defaultHugepagesMountpoint) + assert.NoError(err) + alignMiB := hpBytes >> utils.MibToBytesShift + assert.NotZero(alignMiB) + // Give each of the two nodes two huge pages worth of memory so the total + // is evenly divisible and huge-page aligned on any host. + perNodeMiB := alignMiB * 2 q := &qemu{ config: HypervisorConfig{ DefaultMaxVCPUs: 4, - MemorySize: 1024, + MemorySize: uint32(perNodeMiB * 2), HugePages: true, GuestNUMANodes: []types.GuestNUMANode{ {HostNodes: "0", HostCPUs: "0-1"}, @@ -1366,7 +1376,7 @@ func TestBuildNUMATopologyHugePages(t *testing.T) { assert.Len(nodes, 2) assert.Equal("memory-backend-file", nodes[0].MemBackendType) assert.Equal("/dev/hugepages", nodes[0].MemBackendPath) - assert.Equal("512M", nodes[0].MemSize) + assert.Equal(fmt.Sprintf("%dM", perNodeMiB), nodes[0].MemSize) } func TestBuildNUMATopologyVirtioFS(t *testing.T) {