runtime: add hugepages support

Add hugepages support, port from:
b486387cba

Signed-off-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>
Signed-off-by: bin <bin@hyper.sh>

src/runtime/go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/containernetworking/plugins v1.0.1
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/cri-o/cri-o v1.0.0-rc2.0.20170928185954-3394b3b2d6af
+	github.com/docker/go-units v0.4.0
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-ini/ini v1.28.2
 	github.com/go-openapi/errors v0.18.0

src/runtime/vendor/modules.txt

@@ -134,6 +134,7 @@ github.com/davecgh/go-spew/spew
 # github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c
 github.com/docker/go-events
 # github.com/docker/go-units v0.4.0
+## explicit
 github.com/docker/go-units
 # github.com/fsnotify/fsnotify v1.4.9
 ## explicit

src/runtime/virtcontainers/container.go

@@ -1315,7 +1315,7 @@ func (c *Container) hotplugDrive(ctx context.Context) error {
 			c.rootfsSuffix = ""
 		}
 		// If device mapper device, then fetch the full path of the device
-		devicePath, fsType, err = utils.GetDevicePathAndFsType(dev.mountPoint)
+		devicePath, fsType, _, err = utils.GetDevicePathAndFsTypeOptions(dev.mountPoint)
 		if err != nil {
 			return err
 		}

src/runtime/virtcontainers/device/config/pmem.go

@@ -75,7 +75,7 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) {
 		return nil, fmt.Errorf("backing file %v has not PFN signature", device.HostPath)
 	}
 
-	_, fstype, err := utils.GetDevicePathAndFsType(source)
+	_, fstype, _, err := utils.GetDevicePathAndFsTypeOptions(source)
 	if err != nil {
 		pmemLog.WithError(err).WithField("mount-point", source).Warn("failed to get fstype: using ext4")
 		fstype = "ext4"

src/runtime/virtcontainers/kata_agent.go

@@ -17,6 +17,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/go-units"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
@@ -30,6 +31,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -156,6 +158,16 @@ var kataHostSharedDir = func() string {
 	return defaultKataHostSharedDir
 }
 
+func getPagesizeFromOpt(fsOpts []string) string {
+	// example options array: "rw", "relatime", "seclabel", "pagesize=2M"
+	for _, opt := range fsOpts {
+		if strings.HasPrefix(opt, "pagesize=") {
+			return strings.TrimPrefix(opt, "pagesize=")
+		}
+	}
+	return ""
+}
+
 // Shared path handling:
 // 1. create three directories for each sandbox:
 // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts
@@ -1450,6 +1462,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
 
 	ctrStorages = append(ctrStorages, epheStorages...)
 
+	k.Logger().WithField("ociSpec Hugepage Resources", ociSpec.Linux.Resources.HugepageLimits).Debug("ociSpec HugepageLimit")
+	hugepages, err := k.handleHugepages(ociSpec.Mounts, ociSpec.Linux.Resources.HugepageLimits)
+	if err != nil {
+		return nil, err
+	}
+	ctrStorages = append(ctrStorages, hugepages...)
+
 	localStorages, err := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
 	if err != nil {
 		return nil, err
@@ -1530,6 +1549,71 @@ func buildProcessFromExecID(token string) (*Process, error) {
 	}, nil
 }
 
+// handleHugePages handles hugepages storage by
+// creating a Storage from corresponding source of the mount point
+func (k *kataAgent) handleHugepages(mounts []specs.Mount, hugepageLimits []specs.LinuxHugepageLimit) ([]*grpc.Storage, error) {
+	//Map to hold the total memory of each type of hugepages
+	optionsMap := make(map[int64]string)
+
+	for _, hp := range hugepageLimits {
+		if hp.Limit != 0 {
+			k.Logger().WithFields(logrus.Fields{
+				"Pagesize": hp.Pagesize,
+				"Limit":    hp.Limit,
+			}).Info("hugepage request")
+			//example Pagesize 2MB, 1GB etc. The Limit are in Bytes
+			pageSize, err := units.RAMInBytes(hp.Pagesize)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize to bytes")
+				return nil, err
+			}
+			totalHpSizeStr := strconv.FormatUint(hp.Limit, 10)
+			optionsMap[pageSize] = totalHpSizeStr
+		}
+	}
+
+	var hugepages []*grpc.Storage
+	for idx, mnt := range mounts {
+		if mnt.Type != KataLocalDevType {
+			continue
+		}
+		//HugePages mount Type is Local
+		if _, fsType, fsOptions, _ := utils.GetDevicePathAndFsTypeOptions(mnt.Source); fsType == "hugetlbfs" {
+			k.Logger().WithField("fsOptions", fsOptions).Debug("hugepage mount options")
+			//Find the pagesize from the mountpoint options
+			pagesizeOpt := getPagesizeFromOpt(fsOptions)
+			if pagesizeOpt == "" {
+				return nil, fmt.Errorf("No pagesize option found in filesystem mount options")
+			}
+			pageSize, err := units.RAMInBytes(pagesizeOpt)
+			if err != nil {
+				k.Logger().Error("Unable to convert pagesize from fs mount options to bytes")
+				return nil, err
+			}
+			//Create mount option string
+			options := fmt.Sprintf("pagesize=%s,size=%s", strconv.FormatInt(pageSize, 10), optionsMap[pageSize])
+			k.Logger().WithField("Hugepage options string", options).Debug("hugepage mount options")
+			// Set the mount source path to a path that resides inside the VM
+			mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source))
+			// Set the mount type to "bind"
+			mounts[idx].Type = "bind"
+
+			// Create a storage struct so that kata agent is able to create
+			// hugetlbfs backed volume inside the VM
+			hugepage := &grpc.Storage{
+				Driver:     KataEphemeralDevType,
+				Source:     "nodev",
+				Fstype:     "hugetlbfs",
+				MountPoint: mounts[idx].Source,
+				Options:    []string{options},
+			}
+			hugepages = append(hugepages, hugepage)
+		}
+
+	}
+	return hugepages, nil
+}
+
 // handleEphemeralStorage handles ephemeral storages by
 // creating a Storage from corresponding source of the mount point
 func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) ([]*grpc.Storage, error) {

src/runtime/virtcontainers/kata_agent_test.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -1230,3 +1231,57 @@ func TestSandboxBindMount(t *testing.T) {
 	assert.True(os.IsNotExist(err))
 
 }
+
+func TestHandleHugepages(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("Test disabled as requires root user")
+	}
+
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "hugepages-test")
+	assert.Nil(err)
+	defer os.RemoveAll(dir)
+
+	k := kataAgent{}
+	var mounts []specs.Mount
+	var hugepageLimits []specs.LinuxHugepageLimit
+
+	hugepageDirs := [2]string{"hugepages-1Gi", "hugepages-2Mi"}
+	options := [2]string{"pagesize=1024M", "pagesize=2M"}
+
+	for i := 0; i < 2; i++ {
+		target := path.Join(dir, hugepageDirs[i])
+		err := os.MkdirAll(target, 0777)
+		assert.NoError(err, "Unable to create dir %s", target)
+
+		err = syscall.Mount("nodev", target, "hugetlbfs", uintptr(0), options[i])
+		assert.NoError(err, "Unable to mount %s", target)
+
+		defer syscall.Unmount(target, 0)
+		defer os.RemoveAll(target)
+		mount := specs.Mount{
+			Type:   KataLocalDevType,
+			Source: target,
+		}
+		mounts = append(mounts, mount)
+	}
+
+	hugepageLimits = []specs.LinuxHugepageLimit{
+		{
+			Pagesize: "1GB",
+			Limit:    1073741824,
+		},
+		{
+			Pagesize: "2MB",
+			Limit:    134217728,
+		},
+	}
+
+	hugepages, err := k.handleHugepages(mounts, hugepageLimits)
+
+	assert.NoError(err, "Unable to handle hugepages %v", hugepageLimits)
+	assert.NotNil(hugepages)
+	assert.Equal(len(hugepages), 2)
+
+}

src/runtime/virtcontainers/mount.go

@@ -470,7 +470,7 @@ func IsEphemeralStorage(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType == "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType == "tmpfs" {
 		return true
 	}
 
@@ -485,7 +485,7 @@ func Isk8sHostEmptyDir(path string) bool {
 		return false
 	}
 
-	if _, fsType, _ := utils.GetDevicePathAndFsType(path); fsType != "tmpfs" {
+	if _, fsType, _, _ := utils.GetDevicePathAndFsTypeOptions(path); fsType != "tmpfs" {
 		return true
 	}
 	return false

src/runtime/virtcontainers/utils/utils_linux.go

@@ -96,11 +96,12 @@ const (
 	procDeviceIndex = iota
 	procPathIndex
 	procTypeIndex
+	procOptionIndex
 )
 
-// GetDevicePathAndFsType gets the device for the mount point and the file system type
-// of the mount.
-func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err error) {
+// GetDevicePathAndFsTypeOptions gets the device for the mount point, the file system type
+// and mount options
+func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string, fsOptions []string, err error) {
 	if mountPoint == "" {
 		err = fmt.Errorf("Mount point cannot be empty")
 		return
@@ -134,6 +135,7 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
 		if mountPoint == fields[procPathIndex] {
 			devicePath = fields[procDeviceIndex]
 			fsType = fields[procTypeIndex]
+			fsOptions = strings.Split(fields[procOptionIndex], ",")
 			return
 		}
 	}

src/runtime/virtcontainers/utils/utils_linux_test.go

@@ -6,7 +6,10 @@
 package utils
 
 import (
+	"bytes"
 	"errors"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -34,20 +37,31 @@ func TestFindContextID(t *testing.T) {
 	assert.Error(err)
 }
 
-func TestGetDevicePathAndFsTypeEmptyMount(t *testing.T) {
+func TestGetDevicePathAndFsTypeOptionsEmptyMount(t *testing.T) {
 	assert := assert.New(t)
-	_, _, err := GetDevicePathAndFsType("")
+	_, _, _, err := GetDevicePathAndFsTypeOptions("")
 	assert.Error(err)
 }
 
-func TestGetDevicePathAndFsTypeSuccessful(t *testing.T) {
+func TestGetDevicePathAndFsTypeOptionsSuccessful(t *testing.T) {
 	assert := assert.New(t)
 
-	path, fstype, err := GetDevicePathAndFsType("/proc")
+	cmdStr := "grep ^proc  /proc/mounts"
+	cmd := exec.Command("sh", "-c", cmdStr)
+	output, err := cmd.Output()
+	assert.NoError(err)
+
+	data := bytes.Split(output, []byte(" "))
+	fstypeOut := string(data[2])
+	optsOut := strings.Split(string(data[3]), ",")
+
+	path, fstype, fsOptions, err := GetDevicePathAndFsTypeOptions("/proc")
 	assert.NoError(err)
 
 	assert.Equal(path, "proc")
 	assert.Equal(fstype, "proc")
+	assert.Equal(fstype, fstypeOut)
+	assert.Equal(fsOptions, optsOut)
 }
 
 func TestIsAPVFIOMediatedDeviceFalse(t *testing.T) {