github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-16 23:17:42 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8494 from ChengyuZhu6/kata_virtual_volume

Browse Source 
runtime: Pass `KataVirtualVolume` to the guest as devices in go runtime
This commit is contained in:
Fabiano Fidêncio 2023-11-27 16:02:28 +01:00 committed by GitHub
commit bb4c51a5e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 189 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/runtime/pkg/katautils/create.go
View File

@ -130,7 +130,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
}
if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 {
if rootFs.Source != "" {
if rootFs.Source != "" && !vc.HasOptionPrefix(rootFs.Options, vc.VirtualVolumePrefix) {
realPath, err := ResolvePath(rootFs.Source)
if err != nil {
return nil, vc.Process{}, err

94
src/runtime/virtcontainers/container.go
View File

@ -662,41 +662,9 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
}
}
var stat unix.Stat_t
if err := unix.Stat(c.mounts[i].Source, &stat); err != nil {
return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err)
}
var di *config.DeviceInfo
var err error
// Check if mount is a block device file. If it is, the block device will be attached to the host
// instead of passing this as a shared mount.
if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
di = &config.DeviceInfo{
HostPath: c.mounts[i].Source,
ContainerPath: c.mounts[i].Destination,
DevType: "b",
Major: int64(unix.Major(uint64(stat.Rdev))),
Minor: int64(unix.Minor(uint64(stat.Rdev))),
ReadOnly: c.mounts[i].ReadOnly,
}
} else if isBlockFile && stat.Mode&unix.S_IFMT == unix.S_IFREG {
di = &config.DeviceInfo{
HostPath: c.mounts[i].Source,
ContainerPath: c.mounts[i].Destination,
DevType: "b",
Major: -1,
Minor: 0,
ReadOnly: c.mounts[i].ReadOnly,
}
// Check whether source can be used as a pmem device
} else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil {
c.Logger().WithError(err).
WithField("mount-source", c.mounts[i].Source).
Debug("no loop device")
}
di, err := c.createDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination, c.mounts[i].ReadOnly, isBlockFile)
if err == nil && di != nil {
b, err := c.sandbox.devManager.NewDevice(*di)
if err != nil {
@ -795,6 +763,58 @@ func newContainer(ctx context.Context, sandbox *Sandbox, contConfig *ContainerCo
return c, nil
}
// Create Device Information about the block device
func (c *Container) createDeviceInfo(source, destination string, readonly, isBlockFile bool) (*config.DeviceInfo, error) {
var stat unix.Stat_t
if err := unix.Stat(source, &stat); err != nil {
return nil, fmt.Errorf("stat %q failed: %v", source, err)
}
var di *config.DeviceInfo
var err error
if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
di = &config.DeviceInfo{
HostPath: source,
ContainerPath: destination,
DevType: "b",
Major: int64(unix.Major(uint64(stat.Rdev))),
Minor: int64(unix.Minor(uint64(stat.Rdev))),
ReadOnly: readonly,
}
} else if isBlockFile && stat.Mode&unix.S_IFMT == unix.S_IFREG {
di = &config.DeviceInfo{
HostPath: source,
ContainerPath: destination,
DevType: "b",
Major: -1,
Minor: 0,
ReadOnly: readonly,
}
// Check whether source can be used as a pmem device
} else if di, err = config.PmemDeviceInfo(source, destination); err != nil {
c.Logger().WithError(err).
WithField("mount-source", source).
Debug("no loop device")
}
return di, err
}
// call hypervisor to create device about KataVirtualVolume.
func (c *Container) createVirtualVolumeDevices() ([]config.DeviceInfo, error) {
var deviceInfos []config.DeviceInfo
for _, o := range c.rootFs.Options {
if strings.HasPrefix(o, VirtualVolumePrefix) {
virtVolume, err := types.ParseKataVirtualVolume(strings.TrimPrefix(o, VirtualVolumePrefix))
if err != nil {
return nil, err
}
c.Logger().Infof("KataVirtualVolume volumetype = %s", virtVolume.VolumeType)
}
}
return deviceInfos, nil
}
func (c *Container) createMounts(ctx context.Context) error {
// Create block devices for newly created container
return c.createBlockDevices(ctx)
@ -804,7 +824,13 @@ func (c *Container) createDevices(contConfig *ContainerConfig) error {
// If devices were not found in storage, create Device implementations
// from the configuration. This should happen at create.
var storedDevices []ContainerDevice
for _, info := range contConfig.DeviceInfos {
virtualVolumesDeviceInfos, err := c.createVirtualVolumeDevices()
if err != nil {
return err
}
deviceInfos := append(virtualVolumesDeviceInfos, contConfig.DeviceInfos...)
for _, info := range deviceInfos {
dev, err := c.sandbox.devManager.NewDevice(info)
if err != nil {
return err

3
src/runtime/virtcontainers/fs_share.go
View File

@ -21,7 +21,8 @@ var fsShareTracingTags = map[string]string{
// SharedFile represents the outcome of a host filesystem sharing
// operation.
type SharedFile struct {
storage *grpc.Storage
containerStorages []*grpc.Storage
volumeStorages []*grpc.Storage
guestPath string
}

59
src/runtime/virtcontainers/fs_share_linux.go
View File

@ -15,6 +15,7 @@ import (
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"syscall"
@ -26,6 +27,7 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
@ -464,13 +466,60 @@ func (f *FilesystemShare) shareRootFilesystemWithNydus(ctx context.Context, c *C
f.Logger().Infof("Nydus rootfs info: %#v\n", rootfs)
return &SharedFile{
storage: rootfs,
containerStorages: []*grpc.Storage{rootfs},
guestPath: rootfsGuestPath,
}, nil
}
// handleVirtualVolume processes all `io.katacontainers.volume=` messages in rootFs.Options,
// creating storage, and then aggregates all storages into an array.
func handleVirtualVolume(c *Container) ([]*grpc.Storage, string, error) {
var volumes []*grpc.Storage
var volumeType string
for _, o := range c.rootFs.Options {
if strings.HasPrefix(o, VirtualVolumePrefix) {
virtVolume, err := types.ParseKataVirtualVolume(strings.TrimPrefix(o, VirtualVolumePrefix))
if err != nil {
return nil, "", err
}
volumeType = virtVolume.VolumeType
var vol *grpc.Storage
vol, err = handleVirtualVolumeStorageObject(c, "", virtVolume)
if err != nil {
return nil, "", err
}
if vol != nil {
volumes = append(volumes, vol)
}
}
}
return volumes, volumeType, nil
}
func (f *FilesystemShare) shareRootFilesystemWithVirtualVolume(ctx context.Context, c *Container) (*SharedFile, error) {
guestPath := filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix)
rootFsStorages, _, err := handleVirtualVolume(c)
if err != nil {
return nil, err
}
return &SharedFile{
containerStorages: rootFsStorages,
guestPath: guestPath,
}, nil
}
// func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) {
func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) {
if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
return f.shareRootFilesystemWithVirtualVolume(ctx, c)
}
if c.rootFs.Type == NydusRootFSType {
return f.shareRootFilesystemWithNydus(ctx, c)
}
@ -479,13 +528,13 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container)
if HasOptionPrefix(c.rootFs.Options, annotations.FileSystemLayer) {
path := filepath.Join("/run/kata-containers", c.id, "rootfs")
return &SharedFile{
storage: &grpc.Storage{
containerStorages: []*grpc.Storage{{
MountPoint: path,
Source: "none",
Fstype: c.rootFs.Type,
Driver: kataOverlayDevType,
Options: c.rootFs.Options,
},
}},
guestPath: path,
}, nil
}
@ -550,7 +599,7 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container)
}
return &SharedFile{
storage: rootfsStorage,
containerStorages: []*grpc.Storage{rootfsStorage},
guestPath: rootfsGuestPath,
}, nil
}
@ -565,7 +614,7 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container)
}
return &SharedFile{
storage: nil,
containerStorages: nil,
guestPath: rootfsGuestPath,
}, nil
}

41
src/runtime/virtcontainers/kata_agent.go
View File

@ -69,6 +69,8 @@ const (
NydusRootFSType = "fuse.nydus-overlayfs"
VirtualVolumePrefix = "io.katacontainers.volume="
// enable debug console
kernelParamDebugConsole = "agent.debug_console"
kernelParamDebugConsoleVPort = "agent.debug_console_vport"
@ -89,6 +91,7 @@ var (
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
defaultKataGuestNydusRootDir = "/run/kata-containers/shared/"
defaultKataGuestVirtualVolumedir = "/run/kata-containers/virtual-volumes/"
mountGuestTag = "kataShared"
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
type9pFs = "9p"
@ -1198,6 +1201,10 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
return nil
}
if strings.HasPrefix(dev.ContainerPath, defaultKataGuestVirtualVolumedir) {
continue
}
switch device.DeviceType() {
case config.DeviceBlock:
kataDevice = k.appendBlockDevice(dev, device, c)
@ -1256,12 +1263,17 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
return nil, err
}
if sharedRootfs.storage != nil {
if sharedRootfs.containerStorages != nil {
// Add rootfs to the list of container storage.
// We only need to do this for block based rootfs, as we
ctrStorages = append(ctrStorages, sharedRootfs.containerStorages...)
}
if sharedRootfs.volumeStorages != nil {
// Add volumeStorages to the list of container storage.
// We only need to do this for KataVirtualVolume based rootfs, as we
// want the agent to mount it into the right location
// (kataGuestSharedDir/ctrID/
ctrStorages = append(ctrStorages, sharedRootfs.storage)
ctrStorages = append(ctrStorages, sharedRootfs.volumeStorages...)
}
ociSpec := c.GetPatchedOCISpec()
@ -1536,14 +1548,11 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
return localStorages, nil
}
// handleDeviceBlockVolume handles volume that is block device file
// and DeviceBlock type.
func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) {
func handleBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) {
vol := &grpc.Storage{}
blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
if !ok || blockDrive == nil {
k.Logger().Error("malformed block drive")
return nil, fmt.Errorf("malformed block drive")
}
switch {
@ -1568,6 +1577,22 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De
default:
return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver)
}
return vol, nil
}
// handleVirtualVolumeStorageObject handles KataVirtualVolume that is block device file.
func handleVirtualVolumeStorageObject(c *Container, blockDeviceId string, virtVolume *types.KataVirtualVolume) (*grpc.Storage, error) {
var vol *grpc.Storage = &grpc.Storage{}
return vol, nil
}
// handleDeviceBlockVolume handles volume that is block device file
// and DeviceBlock type.
func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) {
vol, err := handleBlockVolume(c, device)
if err != nil {
return nil, err
}
vol.MountPoint = m.Destination