From 81a8baa5e596d2ce99cbeeee418798a62734a11a Mon Sep 17 00:00:00 2001 From: bin Date: Thu, 23 Dec 2021 14:02:11 +0800 Subject: [PATCH] runtime: add hugepages support Add hugepages support, port from: https://github.com/kata-containers/runtime/pull/3109/commits/b486387cbad09428202154ed7bf2acfdda88f388 Signed-off-by: Pradipta Banerjee Signed-off-by: bin --- src/runtime/go.mod | 1 + src/runtime/vendor/modules.txt | 1 + src/runtime/virtcontainers/container.go | 2 +- .../virtcontainers/device/config/pmem.go | 2 +- src/runtime/virtcontainers/kata_agent.go | 84 +++++++++++++++++++ src/runtime/virtcontainers/kata_agent_test.go | 55 ++++++++++++ src/runtime/virtcontainers/mount.go | 4 +- .../virtcontainers/utils/utils_linux.go | 8 +- .../virtcontainers/utils/utils_linux_test.go | 22 ++++- 9 files changed, 168 insertions(+), 11 deletions(-) diff --git a/src/runtime/go.mod b/src/runtime/go.mod index 07d3932279..9a485b28a6 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -17,6 +17,7 @@ require ( github.com/containernetworking/plugins v1.0.1 github.com/coreos/go-systemd/v22 v22.3.2 github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af + github.com/docker/go-units v0.4.0 github.com/fsnotify/fsnotify v1.4.9 github.com/go-ini/ini v1.28.2 github.com/go-openapi/errors v0.18.0 diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index f663a80330..cde9b39f4a 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -134,6 +134,7 @@ github.com/davecgh/go-spew/spew # github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c github.com/docker/go-events # github.com/docker/go-units v0.4.0 +## explicit github.com/docker/go-units # github.com/fsnotify/fsnotify v1.4.9 ## explicit diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 5c13b08462..87a2a23a50 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -1315,7 +1315,7 @@ func (c *Container) hotplugDrive(ctx context.Context) error { c.rootfsSuffix = "" } // If device mapper device, then fetch the full path of the device - devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint) + devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint) if err != nil { return err } diff --git a/src/runtime/virtcontainers/device/config/pmem.go b/src/runtime/virtcontainers/device/config/pmem.go index 44ea63f729..33ce4fdff4 100644 --- a/src/runtime/virtcontainers/device/config/pmem.go +++ b/src/runtime/virtcontainers/device/config/pmem.go @@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) { return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath) } - _, fstype, err := utils.GetDevicePathAndFsType(source) + _, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source) if err != nil { pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4") fstype = "ext4" diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index dc7b998174..822e84f97e 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -17,6 +17,7 @@ import ( "syscall" "time" + "github.com/docker/go-units" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" @@ -30,6 +31,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/gogo/protobuf/proto" "github.com/opencontainers/runtime-spec/specs-go" @@ -156,6 +158,16 @@ var kataHostSharedDir = func() string { return defaultKataHostSharedDir } +func getPagesizeFromOpt(fsOpts []string) string { + // example options array: "rw", "relatime", "seclabel", "pagesize=2M" + for _, opt := range fsOpts { + if strings.HasPrefix(opt, "pagesize=") { + return strings.TrimPrefix(opt, "pagesize=") + } + } + return "" +} + // Shared path handling: // 1. create three directories for each sandbox: // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts @@ -1450,6 +1462,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co ctrStorages = append(ctrStorages, epheStorages...) + k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit") + hugepages, err := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits) + if err != nil { + return nil, err + } + ctrStorages = append(ctrStorages, hugepages...) + localStorages, err := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) if err != nil { return nil, err @@ -1530,6 +1549,71 @@ func buildProcessFromExecID(token string) (*Process, error) { }, nil } +// handleHugePages handles hugepages storage by +// creating a Storage from corresponding source of the mount point +func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) ([]*grpc.Storage, error) { + //Map to hold the total memory of each type of hugepages + optionsMap := make(map[int64]string) + + for _, hp := range hugepageLimits { + if hp.Limit != 0 { + k.Logger().WithFields(logrus.Fields{ + "Pagesize": hp.Pagesize, + "Limit": hp.Limit, + }).Info("hugepage request") + //example Pagesize 2MB, 1GB etc. The Limit are in Bytes + pageSize, err := units.RAMInBytes(hp.Pagesize) + if err != nil { + k.Logger().Error("Unable to convert pagesize to bytes") + return nil, err + } + totalHpSizeStr := strconv.FormatUint(hp.Limit, 10) + optionsMap[pageSize] = totalHpSizeStr + } + } + + var hugepages []*grpc.Storage + for idx, mnt := range mounts { + if mnt.Type != KataLocalDevType { + continue + } + //HugePages mount Type is Local + if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" { + k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options") + //Find the pagesize from the mountpoint options + pagesizeOpt := getPagesizeFromOpt(fsOptions) + if pagesizeOpt == "" { + return nil, fmt.Errorf("No pagesize option found in filesystem mount options") + } + pageSize, err := units.RAMInBytes(pagesizeOpt) + if err != nil { + k.Logger().Error("Unable to convert pagesize from fs mount options to bytes") + return nil, err + } + //Create mount option string + options := fmt.Sprintf("pagesize=%s,size=%s", strconv.FormatInt(pageSize, 10), optionsMap[pageSize]) + k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options") + // Set the mount source path to a path that resides inside the VM + mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source)) + // Set the mount type to "bind" + mounts[idx].Type = "bind" + + // Create a storage struct so that kata agent is able to create + // hugetlbfs backed volume inside the VM + hugepage := &grpc.Storage{ + Driver: KataEphemeralDevType, + Source: "nodev", + Fstype: "hugetlbfs", + MountPoint: mounts[idx].Source, + Options: []string{options}, + } + hugepages = append(hugepages, hugepage) + } + + } + return hugepages, nil +} + // handleEphemeralStorage handles ephemeral storages by // creating a Storage from corresponding source of the mount point func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) ([]*grpc.Storage, error) { diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index fb596da12d..6475e8b069 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -9,6 +9,7 @@ import ( "bufio" "context" "fmt" + "io/ioutil" "os" "path" "path/filepath" @@ -1230,3 +1231,57 @@ func TestSandboxBindMount(t *testing.T) { assert.True(os.IsNotExist(err)) } + +func TestHandleHugepages(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("Test disabled as requires root user") + } + + assert := assert.New(t) + + dir, err := ioutil.TempDir("", "hugepages-test") + assert.Nil(err) + defer os.RemoveAll(dir) + + k := kataAgent{} + var mounts []specs.Mount + var hugepageLimits []specs.LinuxHugepageLimit + + hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"} + options := [2]string{"pagesize=1024M", "pagesize=2M"} + + for i := 0; i < 2; i++ { + target := path.Join(dir, hugepageDirs[i]) + err := os.MkdirAll(target, 0777) + assert.NoError(err, "Unable to create dir %s", target) + + err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i]) + assert.NoError(err, "Unable to mount %s", target) + + defer syscall.Unmount(target, 0) + defer os.RemoveAll(target) + mount := specs.Mount{ + Type: KataLocalDevType, + Source: target, + } + mounts = append(mounts, mount) + } + + hugepageLimits = []specs.LinuxHugepageLimit{ + { + Pagesize: "1GB", + Limit: 1073741824, + }, + { + Pagesize: "2MB", + Limit: 134217728, + }, + } + + hugepages, err := k.handleHugepages(mounts, hugepageLimits) + + assert.NoError(err, "Unable to handle hugepages %v", hugepageLimits) + assert.NotNil(hugepages) + assert.Equal(len(hugepages), 2) + +} diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index b23bd08793..aca557d461 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -470,7 +470,7 @@ func IsEphemeralStorage(path string) bool { return false } - if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" { + if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" { return true } @@ -485,7 +485,7 @@ func Isk8sHostEmptyDir(path string) bool { return false } - if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" { + if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" { return true } return false diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go index 90e0d631bb..265f10d11b 100644 --- a/src/runtime/virtcontainers/utils/utils_linux.go +++ b/src/runtime/virtcontainers/utils/utils_linux.go @@ -96,11 +96,12 @@ const ( procDeviceIndex = iota procPathIndex procTypeIndex + procOptionIndex ) -// GetDevicePathAndFsType gets the device for the mount point and the file system type -// of the mount. -func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) { +// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type +// and mount options +func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) { if mountPoint == "" { err = fmt.Errorf("Mount point cannot be empty") return @@ -134,6 +135,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e if mountPoint == fields[procPathIndex] { devicePath = fields[procDeviceIndex] fsType = fields[procTypeIndex] + fsOptions = strings.Split(fields[procOptionIndex], ",") return } } diff --git a/src/runtime/virtcontainers/utils/utils_linux_test.go b/src/runtime/virtcontainers/utils/utils_linux_test.go index c7b2b87934..dbf9fde38b 100644 --- a/src/runtime/virtcontainers/utils/utils_linux_test.go +++ b/src/runtime/virtcontainers/utils/utils_linux_test.go @@ -6,7 +6,10 @@ package utils import ( + "bytes" "errors" + "os/exec" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -34,20 +37,31 @@ func TestFindContextID(t *testing.T) { assert.Error(err) } -func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) { +func TestGetDevicePathAndFsTypeOptionsEmptyMount(t *testing.T) { assert := assert.New(t) - _, _, err := GetDevicePathAndFsType("") + _, _, _, err := GetDevicePathAndFsTypeOptions("") assert.Error(err) } -func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) { +func TestGetDevicePathAndFsTypeOptionsSuccessful(t *testing.T) { assert := assert.New(t) - path, fstype, err := GetDevicePathAndFsType("/proc") + cmdStr := "grep ^proc /proc/mounts" + cmd := exec.Command("sh", "-c", cmdStr) + output, err := cmd.Output() + assert.NoError(err) + + data := bytes.Split(output, []byte(" ")) + fstypeOut := string(data[2]) + optsOut := strings.Split(string(data[3]), ",") + + path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc") assert.NoError(err) assert.Equal(path, "proc") assert.Equal(fstype, "proc") + assert.Equal(fstype, fstypeOut) + assert.Equal(fsOptions, optsOut) } func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) {