From d788d4af2ffad0affb3f8860813bb42ec673091d Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Fri, 25 Aug 2023 10:30:56 +0800 Subject: [PATCH 1/8] runtime: Add KataVirtualVolume struct in runtime Add the corresponding data structure in the runtime part according to kata-containers/kata-containers/pull/7698. Signed-off-by: ChengyuZhu6 --- .../virtcontainers/types/virtual_volume.go | 156 +++++++++++ .../types/virtual_volume_test.go | 246 ++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 src/runtime/virtcontainers/types/virtual_volume.go create mode 100644 src/runtime/virtcontainers/types/virtual_volume_test.go diff --git a/src/runtime/virtcontainers/types/virtual_volume.go b/src/runtime/virtcontainers/types/virtual_volume.go new file mode 100644 index 0000000000..2d93519b01 --- /dev/null +++ b/src/runtime/virtcontainers/types/virtual_volume.go @@ -0,0 +1,156 @@ +package types + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/pkg/errors" +) + +const ( + minBlockSize = 1 << 9 + maxBlockSize = 1 << 19 +) + +const ( + KataVirtualVolumeDirectBlockType = "direct_block" + KataVirtualVolumeImageRawBlockType = "image_raw_block" + KataVirtualVolumeLayerRawBlockType = "layer_raw_block" + KataVirtualVolumeImageNydusBlockType = "image_nydus_block" + KataVirtualVolumeLayerNydusBlockType = "layer_nydus_block" + KataVirtualVolumeImageNydusFsType = "image_nydus_fs" + KataVirtualVolumeLayerNydusFsType = "layer_nydus_fs" + KataVirtualVolumeImageGuestPullType = "image_guest_pull" +) + +// DmVerityInfo contains configuration information for DmVerity device. +type DmVerityInfo struct { + HashType string `json:"hashtype"` + Hash string `json:"hash"` + BlockNum uint64 `json:"blocknum"` + Blocksize uint64 `json:"blocksize"` + Hashsize uint64 `json:"hashsize"` + Offset uint64 `json:"offset"` +} + +// DirectAssignedVolume contains meta information for a directly assigned volume. +type DirectAssignedVolume struct { + Metadata map[string]string `json:"metadata"` +} + +// ImagePullVolume contains meta information for pulling an image inside the guest. +type ImagePullVolume struct { + Metadata map[string]string `json:"metadata"` +} + +// NydusImageVolume contains Nydus image volume information. +type NydusImageVolume struct { + Config string `json:"config"` + SnapshotDir string `json:"snapshot_dir"` +} + +// KataVirtualVolume encapsulates information for extra mount options and direct volumes. +type KataVirtualVolume struct { + VolumeType string `json:"volume_type"` + Source string `json:"source,omitempty"` + FSType string `json:"fs_type,omitempty"` + Options []string `json:"options,omitempty"` + DirectVolume *DirectAssignedVolume `json:"direct_volume,omitempty"` + ImagePull *ImagePullVolume `json:"image_pull,omitempty"` + NydusImage *NydusImageVolume `json:"nydus_image,omitempty"` + DmVerity *DmVerityInfo `json:"dm_verity,omitempty"` +} + +func (d *DmVerityInfo) IsValid() error { + err := d.validateHashType() + if err != nil { + return err + } + + if d.BlockNum == 0 || d.BlockNum > uint64(^uint32(0)) { + return fmt.Errorf("Zero block count for DmVerity device %s", d.Hash) + } + + if !isValidBlockSize(d.Blocksize) || !isValidBlockSize(d.Hashsize) { + return fmt.Errorf("Unsupported verity block size: data_block_size = %d, hash_block_size = %d", d.Blocksize, d.Hashsize) + } + + if d.Offset%d.Hashsize != 0 || d.Offset < d.Blocksize*d.BlockNum { + return fmt.Errorf("Invalid hashvalue offset %d for DmVerity device %s", d.Offset, d.Hash) + } + + return nil +} + +func (d *DirectAssignedVolume) IsValid() bool { + return d.Metadata != nil +} + +func (i *ImagePullVolume) IsValid() bool { + return i.Metadata != nil +} + +func (n *NydusImageVolume) IsValid() bool { + return len(n.Config) > 0 || len(n.SnapshotDir) > 0 +} + +func (k *KataVirtualVolume) IsValid() bool { + return len(k.VolumeType) > 0 && + (k.DirectVolume == nil || k.DirectVolume.IsValid()) && + (k.ImagePull == nil || k.ImagePull.IsValid()) && + (k.NydusImage == nil || k.NydusImage.IsValid()) && + (k.DmVerity == nil || k.DmVerity.IsValid() == nil) +} + +func (d *DmVerityInfo) validateHashType() error { + switch strings.ToLower(d.HashType) { + case "sha256": + return d.isValidHash(64, "sha256") + case "sha1": + return d.isValidHash(40, "sha1") + default: + return fmt.Errorf("Unsupported hash algorithm %s for DmVerity device %s", d.HashType, d.Hash) + } +} + +func isValidBlockSize(blockSize uint64) bool { + return minBlockSize <= blockSize && blockSize <= maxBlockSize +} + +func (d *DmVerityInfo) isValidHash(expectedLen int, hashType string) error { + _, err := hex.DecodeString(d.Hash) + if len(d.Hash) != expectedLen || err != nil { + return fmt.Errorf("Invalid hash value %s:%s for DmVerity device with %s", hashType, d.Hash, hashType) + } + return nil +} + +func ParseDmVerityInfo(option string) (*DmVerityInfo, error) { + no := &DmVerityInfo{} + if err := json.Unmarshal([]byte(option), no); err != nil { + return nil, errors.Wrapf(err, "DmVerityInfo json unmarshal err") + } + if err := no.IsValid(); err != nil { + return nil, fmt.Errorf("DmVerityInfo is not correct, %+v; error = %+v", no, err) + } + return no, nil +} + +func ParseKataVirtualVolume(option string) (*KataVirtualVolume, error) { + opt, err := base64.StdEncoding.DecodeString(option) + if err != nil { + return nil, errors.Wrap(err, "KataVirtualVolume base64 decoding err") + } + no := &KataVirtualVolume{} + if err := json.Unmarshal(opt, no); err != nil { + return nil, errors.Wrapf(err, "KataVirtualVolume json unmarshal err") + } + if !no.IsValid() { + return nil, fmt.Errorf("KataVirtualVolume is not correct, %+v", no) + } + + return no, nil +} diff --git a/src/runtime/virtcontainers/types/virtual_volume_test.go b/src/runtime/virtcontainers/types/virtual_volume_test.go new file mode 100644 index 0000000000..6acecf3f34 --- /dev/null +++ b/src/runtime/virtcontainers/types/virtual_volume_test.go @@ -0,0 +1,246 @@ +package types + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDmVerityInfoValidation(t *testing.T) { + TestData := []DmVerityInfo{ + { + HashType: "md5", // "md5" is not a supported hash algorithm + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 3000, // Invalid block size, not a power of 2. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 0, // Invalid block size, less than 512. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 524800, // Invalid block size, greater than 524288. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 3000, // Invalid hash block size, not a power of 2. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 0, // Invalid hash block size, less than 512. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 524800, // Invalid hash block size, greater than 524288. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 0, // Invalid BlockNum, it must be greater than 0. + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 0, // Invalid offset, it must be greater than 0. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8193, // Invalid offset, it must be aligned to 512. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608 - 4096, // Invalid offset, it must be equal to blocksize * BlockNum. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + } + + for _, d := range TestData { + assert.Error(t, d.IsValid()) + } + TestCorrectData := DmVerityInfo{ + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + } + assert.NoError(t, TestCorrectData.IsValid()) +} + +func TestDirectAssignedVolumeValidation(t *testing.T) { + validDirectVolume := DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + } + assert.True(t, validDirectVolume.IsValid()) + + invalidDirectVolume := DirectAssignedVolume{ + Metadata: nil, + } + assert.False(t, invalidDirectVolume.IsValid()) +} + +func TestImagePullVolumeValidation(t *testing.T) { + validImagePull := ImagePullVolume{ + Metadata: map[string]string{"key": "value"}, + } + assert.True(t, validImagePull.IsValid()) + + invalidImagePull := ImagePullVolume{ + Metadata: nil, + } + assert.False(t, invalidImagePull.IsValid()) +} + +func TestNydusImageVolumeValidation(t *testing.T) { + validNydusImage := NydusImageVolume{ + Config: "config_value", + SnapshotDir: "", + } + assert.True(t, validNydusImage.IsValid()) + + invalidNydusImage := NydusImageVolume{ + Config: "", + SnapshotDir: "", + } + assert.False(t, invalidNydusImage.IsValid()) +} + +func TestKataVirtualVolumeValidation(t *testing.T) { + validKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "ext4", + Options: []string{"rw"}, + DirectVolume: &DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + }, + // Initialize other fields + } + assert.True(t, validKataVirtualVolume.IsValid()) + + invalidKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "", + Options: nil, + DirectVolume: &DirectAssignedVolume{ + Metadata: nil, + }, + // Initialize other fields + } + assert.False(t, invalidKataVirtualVolume.IsValid()) +} + +func TestParseDmVerityInfo(t *testing.T) { + // Create a mock valid KataVirtualVolume + validDmVerityInfo := DmVerityInfo{ + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + } + validKataVirtualVolumeJSON, _ := json.Marshal(validDmVerityInfo) + + t.Run("Valid Option", func(t *testing.T) { + volume, err := ParseDmVerityInfo(string(validKataVirtualVolumeJSON)) + assert.NoError(t, err) + assert.NotNil(t, volume) + assert.NoError(t, volume.IsValid()) + }) + + t.Run("Invalid JSON Option", func(t *testing.T) { + volume, err := ParseDmVerityInfo("invalid_json") + assert.Error(t, err) + assert.Nil(t, volume) + }) + +} + +func TestParseKataVirtualVolume(t *testing.T) { + // Create a mock valid KataVirtualVolume + validKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "ext4", + Options: []string{"rw"}, + DirectVolume: &DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + }, + // Initialize other fields + } + validKataVirtualVolumeJSON, _ := json.Marshal(validKataVirtualVolume) + validOption := base64.StdEncoding.EncodeToString(validKataVirtualVolumeJSON) + + t.Run("Valid Option", func(t *testing.T) { + volume, err := ParseKataVirtualVolume(validOption) + + assert.NoError(t, err) + assert.NotNil(t, volume) + assert.True(t, volume.IsValid()) + }) + + t.Run("Invalid JSON Option", func(t *testing.T) { + invalidJSONOption := base64.StdEncoding.EncodeToString([]byte("invalid_json")) + volume, err := ParseKataVirtualVolume(invalidJSONOption) + + assert.Error(t, err) + assert.Nil(t, volume) + }) + + invalidBase64Option := "invalid_base64" + t.Run("Invalid Base64 Option", func(t *testing.T) { + volume, err := ParseKataVirtualVolume(invalidBase64Option) + + assert.Error(t, err) + assert.Nil(t, volume) + }) +} From bedd5364616462c87154d5d75fa28cbe1e976300 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Sep 2023 13:11:44 +0800 Subject: [PATCH 2/8] runtime: add functions to create devices in KataVirtualVolume The snapshotter will place `KataVirtualVolume` information into 'rootfs.options' and commence with the prefix 'io.katacontainers.volume='. The purpose of this commit is to transform the encapsulated KataVirtualVolume data into device information. Fixes #7792 Signed-off-by: ChengyuZhu6 Co-authored-by: Feng Wang Co-authored-by: Samuel Ortiz Co-authored-by: Wedson Almeida Filho --- src/runtime/virtcontainers/container.go | 103 +++++++++++++++-------- src/runtime/virtcontainers/kata_agent.go | 2 + 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 600af63064..314c2b1d73 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -668,41 +668,9 @@ func (c *Container) createBlockDevices(ctx context.Context) error { } } - var stat unix.Stat_t - if err := unix.Stat(c.mounts[i].Source, &stat); err != nil { - return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err) - } - - var di *config.DeviceInfo - var err error - // Check if mount is a block device file. If it is, the block device will be attached to the host // instead of passing this as a shared mount. - if stat.Mode&unix.S_IFMT == unix.S_IFBLK { - di = &config.DeviceInfo{ - HostPath: c.mounts[i].Source, - ContainerPath: c.mounts[i].Destination, - DevType: "b", - Major: int64(unix.Major(uint64(stat.Rdev))), - Minor: int64(unix.Minor(uint64(stat.Rdev))), - ReadOnly: c.mounts[i].ReadOnly, - } - } else if isBlockFile && stat.Mode&unix.S_IFMT == unix.S_IFREG { - di = &config.DeviceInfo{ - HostPath: c.mounts[i].Source, - ContainerPath: c.mounts[i].Destination, - DevType: "b", - Major: -1, - Minor: 0, - ReadOnly: c.mounts[i].ReadOnly, - } - // Check whether source can be used as a pmem device - } else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil { - c.Logger().WithError(err). - WithField("mount-source", c.mounts[i].Source). - Debug("no loop device") - } - + di, err := c.createDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination, c.mounts[i].ReadOnly, isBlockFile) if err == nil && di != nil { b, err := c.sandbox.devManager.NewDevice(*di) if err != nil { @@ -801,6 +769,67 @@ func newContainer(ctx context.Context, sandbox *Sandbox, contConfig *ContainerCo return c, nil } +// Create Device Information about the block device +func (c *Container) createDeviceInfo(source, destination string, readonly, isBlockFile bool) (*config.DeviceInfo, error) { + var stat unix.Stat_t + if err := unix.Stat(source, &stat); err != nil { + return nil, fmt.Errorf("stat %q failed: %v", source, err) + } + + var di *config.DeviceInfo + var err error + + if stat.Mode&unix.S_IFMT == unix.S_IFBLK { + di = &config.DeviceInfo{ + HostPath: source, + ContainerPath: destination, + DevType: "b", + Major: int64(unix.Major(uint64(stat.Rdev))), + Minor: int64(unix.Minor(uint64(stat.Rdev))), + ReadOnly: readonly, + } + } else if isBlockFile && stat.Mode&unix.S_IFMT == unix.S_IFREG { + di = &config.DeviceInfo{ + HostPath: source, + ContainerPath: destination, + DevType: "b", + Major: -1, + Minor: 0, + ReadOnly: readonly, + } + // Check whether source can be used as a pmem device + } else if di, err = config.PmemDeviceInfo(source, destination); err != nil { + c.Logger().WithError(err). + WithField("mount-source", source). + Debug("no loop device") + } + return di, err +} + +// call hypervisor to create device about KataVirtualVolume. +func (c *Container) createVirtualVolumeDevices() ([]config.DeviceInfo, error) { + var deviceInfos []config.DeviceInfo + for _, o := range c.rootFs.Options { + if strings.HasPrefix(o, VirtualVolumePrefix) { + virtVolume, err := types.ParseKataVirtualVolume(strings.TrimPrefix(o, VirtualVolumePrefix)) + if err != nil { + return nil, err + } + if virtVolume.VolumeType == types.KataVirtualVolumeImageRawBlockType || virtVolume.VolumeType == types.KataVirtualVolumeLayerRawBlockType { + di, err := c.createDeviceInfo(virtVolume.Source, virtVolume.Source, true, true) + if err != nil { + return nil, err + } + deviceInfos = append(deviceInfos, *di) + } else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType { + ///TODO implement the logic with pulling image in the guest. + continue + } + } + } + return deviceInfos, nil +} + func (c *Container) createMounts(ctx context.Context) error { // Create block devices for newly created container return c.createBlockDevices(ctx) @@ -810,7 +839,13 @@ func (c *Container) createDevices(contConfig *ContainerConfig) error { // If devices were not found in storage, create Device implementations // from the configuration. This should happen at create. var storedDevices []ContainerDevice - for _, info := range contConfig.DeviceInfos { + virtualVolumesDeviceInfos, err := c.createVirtualVolumeDevices() + if err != nil { + return err + } + deviceInfos := append(virtualVolumesDeviceInfos, contConfig.DeviceInfos...) + + for _, info := range deviceInfos { dev, err := c.sandbox.devManager.NewDevice(info) if err != nil { return err diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 469d8cfc67..b2b29f2e9b 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -69,6 +69,8 @@ const ( NydusRootFSType = "fuse.nydus-overlayfs" + VirtualVolumePrefix = "io.katacontainers.volume=" + // enable debug console kernelParamDebugConsole = "agent.debug_console" kernelParamDebugConsoleVPort = "agent.debug_console_vport" From 29eb2c02d98635c884f31323180768e56569a1fe Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Sep 2023 13:18:23 +0800 Subject: [PATCH 3/8] runtime: extend SharedFile to support mutiple storage devices To enhance the construction and administration of `Katavirtualvolume` storages, this commit expands the 'sharedFile' structure to manage both rootfs storages(`containerStorages`) including `Katavirtualvolume` and other data volumes storages(`volumeStorages`). NOTE: `volumeStorages` is intended for future extensions to support Kubernetes data volumes. Currently, `KataVirtualVolume` is exclusively employed for container rootfs, hence only `containerStorages` is actively utilized. Signed-off-by: ChengyuZhu6 --- src/runtime/virtcontainers/fs_share.go | 5 +++-- src/runtime/virtcontainers/fs_share_linux.go | 20 ++++++++++---------- src/runtime/virtcontainers/kata_agent.go | 13 +++++++++---- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/runtime/virtcontainers/fs_share.go b/src/runtime/virtcontainers/fs_share.go index b5000291cc..3df08368ea 100644 --- a/src/runtime/virtcontainers/fs_share.go +++ b/src/runtime/virtcontainers/fs_share.go @@ -21,8 +21,9 @@ var fsShareTracingTags = map[string]string{ // SharedFile represents the outcome of a host filesystem sharing // operation. type SharedFile struct { - storage *grpc.Storage - guestPath string + containerStorages []*grpc.Storage + volumeStorages []*grpc.Storage + guestPath string } type FilesystemSharer interface { diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index d1dc03e27a..347e63ebfb 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -438,8 +438,8 @@ func (f *FilesystemShare) shareRootFilesystemWithNydus(ctx context.Context, c *C f.Logger().Infof("Nydus rootfs info: %#v\n", rootfs) return &SharedFile{ - storage: rootfs, - guestPath: rootfsGuestPath, + containerStorages: []*grpc.Storage{rootfs}, + guestPath: rootfsGuestPath, }, nil } @@ -451,8 +451,8 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) // so there is no Rootfs.Target. if f.sandbox.config.ServiceOffload && c.rootFs.Target == "" { return &SharedFile{ - storage: nil, - guestPath: rootfsGuestPath, + containerStorages: nil, + guestPath: rootfsGuestPath, }, nil } @@ -463,13 +463,13 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) if HasOptionPrefix(c.rootFs.Options, annotations.FileSystemLayer) { path := filepath.Join("/run/kata-containers", c.id, "rootfs") return &SharedFile{ - storage: &grpc.Storage{ + containerStorages: []*grpc.Storage{{ MountPoint: path, Source: "none", Fstype: c.rootFs.Type, Driver: kataOverlayDevType, Options: c.rootFs.Options, - }, + }}, guestPath: path, }, nil } @@ -534,8 +534,8 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) } return &SharedFile{ - storage: rootfsStorage, - guestPath: rootfsGuestPath, + containerStorages: []*grpc.Storage{rootfsStorage}, + guestPath: rootfsGuestPath, }, nil } @@ -549,8 +549,8 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) } return &SharedFile{ - storage: nil, - guestPath: rootfsGuestPath, + containerStorages: nil, + guestPath: rootfsGuestPath, }, nil } diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index b2b29f2e9b..07acff21f2 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1258,12 +1258,17 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co return nil, err } - if sharedRootfs.storage != nil { + if sharedRootfs.containerStorages != nil { // Add rootfs to the list of container storage. - // We only need to do this for block based rootfs, as we + ctrStorages = append(ctrStorages, sharedRootfs.containerStorages...) + } + + if sharedRootfs.volumeStorages != nil { + // Add volumeStorages to the list of container storage. + // We only need to do this for KataVirtualVolume based rootfs, as we // want the agent to mount it into the right location - // (kataGuestSharedDir/ctrID/ - ctrStorages = append(ctrStorages, sharedRootfs.storage) + + ctrStorages = append(ctrStorages, sharedRootfs.volumeStorages...) } ociSpec := c.GetPatchedOCISpec() From 5ad3eba8b169a8027803ebff7baf9606f13eb4b8 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Sep 2023 13:29:16 +0800 Subject: [PATCH 4/8] runtime: redefine and add functions to handle VirtualVolume to storage 1) Extract function `handleBlockVolume` to create Storage only. 2) Add functions to handle KataVirtualVolume device and construct corresponding storages. Signed-off-by: ChengyuZhu6 --- src/runtime/virtcontainers/kata_agent.go | 145 ++++++++++++++++------- 1 file changed, 105 insertions(+), 40 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 07acff21f2..cbd76b1b45 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -83,42 +83,44 @@ const ( type customRequestTimeoutKeyType struct{} var ( - checkRequestTimeout = 30 * time.Second - defaultRequestTimeout = 60 * time.Second - imageRequestTimeout = 60 * time.Second - remoteRequestTimeout = 300 * time.Second - customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" - defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" - typeVirtioFS = "virtiofs" - typeOverlayFS = "overlay" - kata9pDevType = "9p" - kataMmioBlkDevType = "mmioblk" - kataBlkDevType = "blk" - kataBlkCCWDevType = "blk-ccw" - kataSCSIDevType = "scsi" - kataNvdimmDevType = "nvdimm" - kataVirtioFSDevType = "virtio-fs" - kataOverlayDevType = "overlayfs" - kataWatchableBindDevType = "watchable-bind" - kataVfioPciDevType = "vfio-pci" // VFIO device to used as VFIO in the container - kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO device for consumption by the guest kernel - kataVfioApDevType = "vfio-ap" - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} - sharedDirVirtioFSOptions = []string{} - sharedDirVirtioFSDaxOptions = "dax" - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) - grpcMaxDataSize = int64(1024 * 1024) - localDirOptions = []string{"mode=0777"} - maxHostnameLen = 64 - GuestDNSFile = "/etc/resolv.conf" + checkRequestTimeout = 30 * time.Second + defaultRequestTimeout = 60 * time.Second + imageRequestTimeout = 60 * time.Second + remoteRequestTimeout = 300 * time.Second + customRequestTimeoutKey = customRequestTimeoutKeyType(struct{}{}) + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataGuestVirtualVolumedir = "/run/kata-containers/virtual-volumes/" + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" + typeVirtioFS = "virtiofs" + typeOverlayFS = "overlay" + kata9pDevType = "9p" + kataMmioBlkDevType = "mmioblk" + kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" + kataSCSIDevType = "scsi" + kataNvdimmDevType = "nvdimm" + kataVirtioFSDevType = "virtio-fs" + kataOverlayDevType = "overlayfs" + kataWatchableBindDevType = "watchable-bind" + kataVfioPciDevType = "vfio-pci" // VFIO device to used as VFIO in the container + kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO device for consumption by the guest kernel + kataVfioApDevType = "vfio-ap" + kataDmVerityBlkDevType = "dmverity" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} + sharedDirVirtioFSOptions = []string{} + sharedDirVirtioFSDaxOptions = "dax" + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) + grpcMaxDataSize = int64(1024 * 1024) + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ) const ( @@ -1544,14 +1546,36 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r return localStorages, nil } -// handleDeviceBlockVolume handles volume that is block device file -// and DeviceBlock type. -func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) { +// Add the source block type to DriverOptions in the volume with dm-verity +func handleDmVerityBlockVolume(driverType, source string, verityInfo *types.DmVerityInfo, vol *grpc.Storage) (*grpc.Storage, error) { + no, err := json.Marshal(verityInfo) + if err != nil { + return nil, err + } + vol.Driver = kataDmVerityBlkDevType + vol.DriverOptions = append(vol.DriverOptions, "verity_info="+string(no)) + switch driverType { + case kataNvdimmDevType: + vol.DriverOptions = append(vol.DriverOptions, "source_type=pmem") + case kataBlkCCWDevType: + vol.DriverOptions = append(vol.DriverOptions, "source_type=virtio_ccw") + case kataBlkDevType: + vol.DriverOptions = append(vol.DriverOptions, "source_type=virtio_pci") + case kataMmioBlkDevType: + vol.DriverOptions = append(vol.DriverOptions, "source_type=virtio_mmio") + case kataSCSIDevType: + vol.DriverOptions = append(vol.DriverOptions, "source_type=scsi") + } + vol.Options = []string{"ro"} + vol.MountPoint = filepath.Join(defaultKataGuestVirtualVolumedir, "verity", verityInfo.Hash) + return vol, nil +} + +func handleBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) { vol := &grpc.Storage{} blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive) if !ok || blockDrive == nil { - k.Logger().Error("malformed block drive") return nil, fmt.Errorf("malformed block drive") } switch { @@ -1576,6 +1600,47 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De default: return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver) } + return vol, nil +} + +// handleVirtualVolumeStorageObject handles KataVirtualVolume that is block device file. +func handleVirtualVolumeStorageObject(c *Container, blockDeviceId string, virtVolume *types.KataVirtualVolume) (*grpc.Storage, error) { + var vol *grpc.Storage + if virtVolume.VolumeType == types.KataVirtualVolumeImageRawBlockType || virtVolume.VolumeType == types.KataVirtualVolumeLayerRawBlockType { + device := c.sandbox.devManager.GetDeviceByID(blockDeviceId) + if device == nil { + return nil, fmt.Errorf("Failed to find device by id (id=%s) in handleVirtualVolumeStorageObject", blockDeviceId) + } + var err error + vol, err = handleBlockVolume(c, device) + if err != nil { + return nil, err + } + filename := b64.URLEncoding.EncodeToString([]byte(vol.Source)) + vol.MountPoint = filepath.Join(defaultKataGuestVirtualVolumedir, filename) + + //convert block storage to dmverity storage if dm-verity info is available + if virtVolume.DmVerity != nil { + vol, err = handleDmVerityBlockVolume(vol.Driver, virtVolume.Source, virtVolume.DmVerity, vol) + if err != nil { + return nil, err + } + } + } else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType { + ///TODO implement the logic with pulling image in the guest. + return nil, nil + } + + return vol, nil +} + +// handleDeviceBlockVolume handles volume that is block device file +// and DeviceBlock type. +func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) { + vol, err := handleBlockVolume(c, device) + if err != nil { + return nil, err + } vol.MountPoint = m.Destination From e36c2b62497b5dcf3a89037c0f0f6aca92c6109b Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Sep 2023 13:34:30 +0800 Subject: [PATCH 5/8] runtime: support to create VirtualVolume rootfs storages 1) Creating storage for each `extraoption` in rootFs.Options, and then aggregates all storages into `containerStorages`. 2) Creating storage for other data volumes and push them into `volumeStorages`. Signed-off-by: ChengyuZhu6 --- src/runtime/pkg/katautils/create.go | 2 +- src/runtime/virtcontainers/fs_share_linux.go | 76 ++++++++++++++++++++ src/runtime/virtcontainers/kata_agent.go | 4 ++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go index 3c3bf05c8d..bcc7d8ce7f 100644 --- a/src/runtime/pkg/katautils/create.go +++ b/src/runtime/pkg/katautils/create.go @@ -129,7 +129,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo } if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 { - if rootFs.Source != "" { + if rootFs.Source != "" && !vc.HasOptionPrefix(rootFs.Options, vc.VirtualVolumePrefix) { realPath, err := ResolvePath(rootFs.Source) if err != nil { return nil, vc.Process{}, err diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index 347e63ebfb..06c21c3afa 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -27,6 +27,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -443,6 +444,77 @@ func (f *FilesystemShare) shareRootFilesystemWithNydus(ctx context.Context, c *C }, nil } +// handleVirtualVolume processes each `extraoption` in rootFs.Options, +// creating storage, and then aggregates all storages into an array. +func handleVirtualVolume(c *Container) ([]*grpc.Storage, string, error) { + var volumes []*grpc.Storage + var volumeType string + + for _, o := range c.rootFs.Options { + if strings.HasPrefix(o, VirtualVolumePrefix) { + virtVolume, err := types.ParseKataVirtualVolume(strings.TrimPrefix(o, VirtualVolumePrefix)) + if err != nil { + return nil, "", err + } + + volumeType = virtVolume.VolumeType + var vol *grpc.Storage + if virtVolume.VolumeType == types.KataVirtualVolumeImageRawBlockType || virtVolume.VolumeType == types.KataVirtualVolumeLayerRawBlockType { + for i, d := range c.devices { + if d.ContainerPath == virtVolume.Source { + vol, err = handleVirtualVolumeStorageObject(c, d.ID, virtVolume) + if err != nil { + return nil, "", err + } + c.devices[i].ContainerPath = vol.MountPoint + vol.Fstype = virtVolume.FSType + vol.Options = append(vol.Options, virtVolume.Options...) + break + } + } + } + if vol != nil { + volumes = append(volumes, vol) + } + } + } + + return volumes, volumeType, nil +} + +func (f *FilesystemShare) shareRootFilesystemWithVirtualVolume(ctx context.Context, c *Container) (*SharedFile, error) { + kataGuestDir := filepath.Join(defaultKataGuestVirtualVolumedir, "containers") + guestPath := filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix) + rootFsStorages, volumeType, err := handleVirtualVolume(c) + if err != nil { + return nil, err + } + + if volumeType == types.KataVirtualVolumeImageRawBlockType || volumeType == types.KataVirtualVolumeLayerRawBlockType { + rootfs := &grpc.Storage{} + rootfs.MountPoint = guestPath + overlayDirDriverOption := "io.katacontainers.volume.overlayfs.create_directory" + rootfs.Source = typeOverlayFS + rootfs.Fstype = typeOverlayFS + rootfs.Driver = kataOverlayDevType + for _, v := range rootFsStorages { + rootfs.Options = append(rootfs.Options, fmt.Sprintf("%s=%s", lowerDir, v.MountPoint)) + } + rootfsUpperDir := filepath.Join(kataGuestDir, c.id, "fs") + rootfsWorkDir := filepath.Join(kataGuestDir, c.id, "work") + rootfs.DriverOptions = append(rootfs.DriverOptions, fmt.Sprintf("%s=%s", overlayDirDriverOption, rootfsUpperDir)) + rootfs.DriverOptions = append(rootfs.DriverOptions, fmt.Sprintf("%s=%s", overlayDirDriverOption, rootfsWorkDir)) + rootfs.Options = append(rootfs.Options, fmt.Sprintf("%s=%s", upperDir, rootfsUpperDir)) + rootfs.Options = append(rootfs.Options, fmt.Sprintf("%s=%s", workDir, rootfsWorkDir)) + rootFsStorages = append(rootFsStorages, rootfs) + f.Logger().Infof("verity rootfs info: %#v\n", rootfs) + } + return &SharedFile{ + containerStorages: rootFsStorages, + guestPath: guestPath, + }, nil +} + // func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) { func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) { rootfsGuestPath := filepath.Join(kataGuestSharedDir(), c.id, c.rootfsSuffix) @@ -456,6 +528,10 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) }, nil } + if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) { + return f.shareRootFilesystemWithVirtualVolume(ctx, c) + } + if c.rootFs.Type == NydusRootFSType { return f.shareRootFilesystemWithNydus(ctx, c) } diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index cbd76b1b45..21b99d3579 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1202,6 +1202,10 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr return nil } + if strings.HasPrefix(dev.ContainerPath, defaultKataGuestVirtualVolumedir) { + continue + } + switch device.DeviceType() { case config.DeviceBlock: kataDevice = k.appendBlockDevice(dev, device, c) From 72c9f62b706bc8476c6df17b3cc5c102a71a0a61 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Fri, 1 Sep 2023 00:50:40 +0800 Subject: [PATCH 6/8] agent: introduce DmVerityHandler to support dm-verity volume We utilize the KataVirtualVolume which storing the dm-verity info and the path of disk image on the host supplied by snapshotter as an integral part of `CreateContainer`. Within this process, we copy the verity info and the disk image path to mount slice to create a block device by virtio-blk. Then storing the `lowerdir` in rootfs.storage which is the mountpoint of the verity path through `CreateContainerRequest`. To maintain clarity and avoid any need for modification to the `VirtioBlkPciHandler`,we introduce the `DmVerityHandler`. This dedicated handler is responsible for calling image-rs to create verity device and mount the device to the `lowerdir` within the guest environment. Signed-off-by: Jiang Liu Signed-off-by: ChengyuZhu6 --- src/agent/src/storage/block_handler.rs | 105 ++++++++++++---- src/agent/src/storage/dm_verity.rs | 165 +++++++++++++++++++++++++ src/agent/src/storage/mod.rs | 4 + src/libs/kata-types/src/lib.rs | 3 + src/libs/kata-types/src/volume.rs | 22 ++++ 5 files changed, 272 insertions(+), 27 deletions(-) create mode 100644 src/agent/src/storage/dm_verity.rs create mode 100644 src/libs/kata-types/src/volume.rs diff --git a/src/agent/src/storage/block_handler.rs b/src/agent/src/storage/block_handler.rs index 60330253ce..6af326a555 100644 --- a/src/agent/src/storage/block_handler.rs +++ b/src/agent/src/storage/block_handler.rs @@ -27,19 +27,29 @@ use crate::{ccw, device::get_virtio_blk_ccw_device_name}; #[derive(Debug)] pub struct VirtioBlkMmioHandler {} -#[async_trait::async_trait] -impl StorageHandler for VirtioBlkMmioHandler { - #[instrument] - async fn create_device( - &self, - storage: Storage, - ctx: &mut StorageContext, - ) -> Result> { +impl VirtioBlkMmioHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { if !Path::new(&storage.source).exists() { get_virtio_mmio_device_name(ctx.sandbox, &storage.source) .await .context("failed to get mmio device name")?; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for VirtioBlkMmioHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -48,14 +58,11 @@ impl StorageHandler for VirtioBlkMmioHandler { #[derive(Debug)] pub struct VirtioBlkPciHandler {} -#[async_trait::async_trait] -impl StorageHandler for VirtioBlkPciHandler { - #[instrument] - async fn create_device( - &self, - mut storage: Storage, - ctx: &mut StorageContext, - ) -> Result> { +impl VirtioBlkPciHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { // If hot-plugged, get the device node path based on the PCI path // otherwise use the virt path provided in Storage Source if storage.source.starts_with("/dev") { @@ -71,6 +78,19 @@ impl StorageHandler for VirtioBlkPciHandler { storage.source = dev_path; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for VirtioBlkPciHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -79,6 +99,21 @@ impl StorageHandler for VirtioBlkPciHandler { #[derive(Debug)] pub struct VirtioBlkCcwHandler {} +impl VirtioBlkCcwHandler { + pub async fn update_device_path( + _storage: &mut Storage, + _ctx: &mut StorageContext<'_>, + ) -> Result<()> { + #[cfg(target_arch = "s390x")] + { + let ccw_device = ccw::Device::from_str(&_storage.source)?; + let dev_path = get_virtio_blk_ccw_device_name(_ctx.sandbox, &ccw_device).await?; + _storage.source = dev_path; + } + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for VirtioBlkCcwHandler { #[cfg(target_arch = "s390x")] @@ -88,9 +123,7 @@ impl StorageHandler for VirtioBlkCcwHandler { mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - let ccw_device = ccw::Device::from_str(&storage.source)?; - let dev_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?; - storage.source = dev_path; + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -109,6 +142,18 @@ impl StorageHandler for VirtioBlkCcwHandler { #[derive(Debug)] pub struct ScsiHandler {} +impl ScsiHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + // Retrieve the device path from SCSI address. + let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?; + storage.source = dev_path; + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for ScsiHandler { #[instrument] @@ -117,10 +162,7 @@ impl StorageHandler for ScsiHandler { mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - // Retrieve the device path from SCSI address. - let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?; - storage.source = dev_path; - + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } @@ -129,17 +171,26 @@ impl StorageHandler for ScsiHandler { #[derive(Debug)] pub struct PmemHandler {} +impl PmemHandler { + pub async fn update_device_path( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + // Retrieve the device for pmem storage + wait_for_pmem_device(ctx.sandbox, &storage.source).await?; + Ok(()) + } +} + #[async_trait::async_trait] impl StorageHandler for PmemHandler { #[instrument] async fn create_device( &self, - storage: Storage, + mut storage: Storage, ctx: &mut StorageContext, ) -> Result> { - // Retrieve the device for pmem storage - wait_for_pmem_device(ctx.sandbox, &storage.source).await?; - + Self::update_device_path(&mut storage, ctx).await?; let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) } diff --git a/src/agent/src/storage/dm_verity.rs b/src/agent/src/storage/dm_verity.rs new file mode 100644 index 0000000000..e177538fc4 --- /dev/null +++ b/src/agent/src/storage/dm_verity.rs @@ -0,0 +1,165 @@ +// Copyright (c) 2023 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use image_rs::verity::{create_dmverity_device, destroy_dmverity_device}; +use kata_sys_util::mount::create_mount_destination; +use kata_types::mount::{DmVerityInfo, StorageDevice}; +use kata_types::volume::{ + KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE, KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM, KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW, KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO, + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI, +}; +use protocols::agent::Storage; +use slog::Logger; +use tracing::instrument; + +use crate::storage::block_handler::{ + PmemHandler, ScsiHandler, VirtioBlkCcwHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler, +}; +use crate::storage::{common_storage_handler, StorageContext, StorageHandler}; + +use super::StorageDeviceGeneric; + +#[derive(Debug)] +pub struct DmVerityHandler {} + +impl DmVerityHandler { + fn get_dm_verity_info(storage: &Storage) -> Result { + for option in storage.driver_options.iter() { + if let Some((key, value)) = option.split_once('=') { + if key == KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO { + let verity_info: DmVerityInfo = serde_json::from_str(value)?; + return Ok(verity_info); + } + } + } + + Err(anyhow!("missing DmVerity information for DmVerity volume")) + } + + async fn update_source_device( + storage: &mut Storage, + ctx: &mut StorageContext<'_>, + ) -> Result<()> { + for option in storage.driver_options.clone() { + if let Some((key, value)) = option.split_once('=') { + if key == KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE { + match value { + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI => { + VirtioBlkPciHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO => { + VirtioBlkMmioHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW => { + VirtioBlkCcwHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI => { + ScsiHandler::update_device_path(storage, ctx).await?; + } + KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM => { + PmemHandler::update_device_path(storage, ctx).await?; + } + _ => {} + } + } + } + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl StorageHandler for DmVerityHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + Self::update_source_device(&mut storage, ctx).await?; + create_mount_destination(&storage.source, &storage.mount_point, "", &storage.fstype) + .context("Could not create mountpoint")?; + + let verity_info = Self::get_dm_verity_info(&storage)?; + let verity_info = serde_json::to_string(&verity_info) + .map_err(|e| anyhow!("failed to serialize dm_verity info, {}", e))?; + let verity_device_path = create_dmverity_device(&verity_info, Path::new(storage.source())) + .context("create device with dm-verity enabled")?; + storage.source = verity_device_path; + common_storage_handler(ctx.logger, &storage)?; + + Ok(Arc::new(DmVerityDevice { + common: StorageDeviceGeneric::new(storage.mount_point), + verity_device_path: storage.source, + logger: ctx.logger.clone(), + })) + } +} + +struct DmVerityDevice { + common: StorageDeviceGeneric, + verity_device_path: String, + logger: Logger, +} + +impl StorageDevice for DmVerityDevice { + fn path(&self) -> Option<&str> { + self.common.path() + } + + fn cleanup(&self) -> Result<()> { + self.common.cleanup().context("clean up dm-verity volume")?; + let device_path = &self.verity_device_path; + debug!( + self.logger, + "destroy verity device path = {:?}", device_path + ); + destroy_dmverity_device(device_path)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use kata_types::{mount::DmVerityInfo, volume::KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO}; + use protocols::agent::Storage; + + use crate::storage::dm_verity::DmVerityHandler; + + #[test] + fn test_get_dm_verity_info() { + let verity_info = DmVerityInfo { + hashtype: "sha256".to_string(), + hash: "d86104eee715a1b59b62148641f4ca73edf1be3006c4d481f03f55ac05640570".to_string(), + blocknum: 2361, + blocksize: 512, + hashsize: 4096, + offset: 1212416, + }; + + let verity_info_str = serde_json::to_string(&verity_info); + assert!(verity_info_str.is_ok()); + + let storage = Storage { + driver: KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO.to_string(), + driver_options: vec![format!("verity_info={}", verity_info_str.ok().unwrap())], + ..Default::default() + }; + + match DmVerityHandler::get_dm_verity_info(&storage) { + Ok(result) => { + assert_eq!(verity_info, result); + } + Err(e) => panic!("err = {}", e), + } + } +} diff --git a/src/agent/src/storage/mod.rs b/src/agent/src/storage/mod.rs index f312bbd83b..a8a3b3717a 100644 --- a/src/agent/src/storage/mod.rs +++ b/src/agent/src/storage/mod.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use kata_sys_util::mount::{create_mount_destination, parse_mount_options}; use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG}; +use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY; use nix::unistd::{Gid, Uid}; use protocols::agent::Storage; use protocols::types::FSGroupChangePolicy; @@ -22,6 +23,7 @@ use tracing::instrument; use self::bind_watcher_handler::BindWatcherHandler; use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler}; +use self::dm_verity::DmVerityHandler; use self::ephemeral_handler::EphemeralHandler; use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler}; use self::local_handler::LocalHandler; @@ -37,6 +39,7 @@ pub use self::ephemeral_handler::update_ephemeral_mounts; mod bind_watcher_handler; mod block_handler; +mod dm_verity; mod ephemeral_handler; mod fs_handler; mod local_handler; @@ -145,6 +148,7 @@ lazy_static! { manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap(); manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap(); manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap(); + manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap(); manager }; } diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index 5eb407561c..57090028a3 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -29,6 +29,9 @@ pub mod k8s; /// Constants and data types related to mount point. pub mod mount; +/// Constants and data types related to data volumes. +pub mod volume; + pub(crate) mod utils; /// hypervisor capabilities diff --git a/src/libs/kata-types/src/volume.rs b/src/libs/kata-types/src/volume.rs new file mode 100644 index 0000000000..2a6de62238 --- /dev/null +++ b/src/libs/kata-types/src/volume.rs @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// Volume to support dm-verity over block devices. +pub const KATA_VOLUME_TYPE_DMVERITY: &str = "dmverity"; + +/// Key to identify dmverity information in `Storage.driver_options` +pub const KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO: &str = "verity_info"; +/// Key to identify type of source device in `Storage.driver_options` +pub const KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE: &str = "source_type"; +/// Source device of dmverity volume is a Virtio PCI device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI: &str = "virtio_pci"; +/// Source device of dmverity volume is a Virtio MMIO device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO: &str = "virtio_mmio"; +/// Source device of dmverity volume is a Virtio CCW device +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW: &str = "virtio_ccw"; +/// Source device of dmverity volume is a SCSI disk. +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI: &str = "scsi"; +/// Source device of dmverity volume is a pmem disk. +pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM: &str = "pmem"; From 622bd4e37064e3200591972a04e94f5cd2725bc9 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Fri, 1 Sep 2023 01:01:34 +0800 Subject: [PATCH 7/8] agent: create directories to mount filesystem by overlay When creating a container with a raw disk image using virtio-blk, the guest does not have the upper directory and worker directory present. Therefore, it is necessary to create these directories before mounting the filesystem with overlay. Signed-off-by: ChengyuZhu6 --- src/agent/src/storage/fs_handler.rs | 10 ++++++++++ src/libs/kata-sys-util/src/mount.rs | 9 ++++++--- src/libs/kata-types/src/volume.rs | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/agent/src/storage/fs_handler.rs b/src/agent/src/storage/fs_handler.rs index fce59c0b14..009247b5a6 100644 --- a/src/agent/src/storage/fs_handler.rs +++ b/src/agent/src/storage/fs_handler.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use kata_types::mount::StorageDevice; +use kata_types::volume::KATA_VOLUME_OVERLAYFS_CREATE_DIR; use protocols::agent::Storage; use tracing::instrument; @@ -50,6 +51,15 @@ impl StorageHandler for OverlayfsHandler { .options .push(format!("workdir={}", work.to_string_lossy())); } + let overlay_create_dir_prefix = &(KATA_VOLUME_OVERLAYFS_CREATE_DIR.to_string() + "="); + for driver_option in &storage.driver_options { + if let Some(dir) = driver_option + .as_str() + .strip_prefix(overlay_create_dir_prefix) + { + fs::create_dir_all(dir).context("Failed to create directory")?; + } + } let path = common_storage_handler(ctx.logger, &storage)?; new_device(path) diff --git a/src/libs/kata-sys-util/src/mount.rs b/src/libs/kata-sys-util/src/mount.rs index 873db5f5b9..4ab559204c 100644 --- a/src/libs/kata-sys-util/src/mount.rs +++ b/src/libs/kata-sys-util/src/mount.rs @@ -399,12 +399,15 @@ pub fn parse_mount_options>(options: &[T]) -> Result<(MsFlags, Str let mut data: Vec = Vec::new(); for opt in options.iter() { - if opt.as_ref() == "loop" { + let opt_str = opt.as_ref(); + if matches!(opt_str, "loop") { return Err(Error::InvalidMountOption("loop".to_string())); - } else if let Some(v) = parse_mount_flags(flags, opt.as_ref()) { + } else if let Some(v) = parse_mount_flags(flags, opt_str) { flags = v; + } else if opt_str.starts_with("io.katacontainers.") { + continue; } else { - data.push(opt.as_ref().to_string()); + data.push(opt_str.to_string()); } } diff --git a/src/libs/kata-types/src/volume.rs b/src/libs/kata-types/src/volume.rs index 2a6de62238..6c7db968ab 100644 --- a/src/libs/kata-types/src/volume.rs +++ b/src/libs/kata-types/src/volume.rs @@ -20,3 +20,7 @@ pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW: &str = "virtio_ccw"; pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI: &str = "scsi"; /// Source device of dmverity volume is a pmem disk. pub const KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM: &str = "pmem"; + +/// Key to indentify directory creation in `Storage.driver_options`. +pub const KATA_VOLUME_OVERLAYFS_CREATE_DIR: &str = + "io.katacontainers.volume.overlayfs.create_directory"; From a533c974f9667608cc2c519da75764b692ee31ae Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Fri, 1 Sep 2023 02:01:42 +0800 Subject: [PATCH 8/8] agent: enable `verity` feature in image-rs update image-rs to support verity feature Signed-off-by: ChengyuZhu6 --- src/agent/Cargo.lock | 67 ++++++++++++++++++++++++++++++++++++++++++-- src/agent/Cargo.toml | 4 +-- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 1bd9336229..cd5b8b499f 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -1095,6 +1095,34 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "devicemapper" +version = "0.33.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a9fd602a98d192f7662a1f4c4cf6920a1b454c3a9e724f6490cf8e30910114" +dependencies = [ + "bitflags", + "devicemapper-sys", + "env_logger", + "lazy_static", + "log", + "nix 0.26.2", + "rand 0.8.5", + "retry", + "semver", + "serde", +] + +[[package]] +name = "devicemapper-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0b0f9d16560f830ae6e90b769017333c4561d2c84f39e7aa7d935d2e7bcbc4c" +dependencies = [ + "bindgen", + "nix 0.26.2", +] + [[package]] name = "diff" version = "0.1.13" @@ -1335,6 +1363,19 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "errno" version = "0.2.8" @@ -1817,6 +1858,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.26" @@ -1907,13 +1954,14 @@ dependencies = [ [[package]] name = "image-rs" version = "0.1.0" -source = "git+https://github.com/confidential-containers/guest-components?rev=53ddd632424432077e95d3901deb64727be0b4c1#53ddd632424432077e95d3901deb64727be0b4c1" +source = "git+https://github.com/confidential-containers/guest-components?rev=e9944577d1f61060c51d48890359a5467d519a29#e9944577d1f61060c51d48890359a5467d519a29" dependencies = [ "anyhow", "async-compression", "async-trait", "base64 0.21.2", "cfg-if 1.0.0", + "devicemapper", "flate2", "futures", "futures-util", @@ -2811,7 +2859,7 @@ dependencies = [ [[package]] name = "ocicrypt-rs" version = "0.1.0" -source = "git+https://github.com/confidential-containers/guest-components?rev=53ddd632424432077e95d3901deb64727be0b4c1#53ddd632424432077e95d3901deb64727be0b4c1" +source = "git+https://github.com/confidential-containers/guest-components?rev=e9944577d1f61060c51d48890359a5467d519a29#e9944577d1f61060c51d48890359a5467d519a29" dependencies = [ "aes 0.8.3", "anyhow", @@ -3812,6 +3860,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "retry" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac95c60a949a63fd2822f4964939662d8f2c16c4fa0624fd954bc6e703b9a3f6" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -4110,6 +4167,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" + [[package]] name = "sequoia-openpgp" version = "1.14.0" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 876a60727d..f7a273d09b 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -74,8 +74,8 @@ clap = { version = "3.0.1", features = ["derive"] } openssl = { version = "0.10.38", features = ["vendored"] } # Image pull/decrypt -image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "53ddd632424432077e95d3901deb64727be0b4c1", default-features = false, features = [ - "kata-cc-native-tls", +image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "e9944577d1f61060c51d48890359a5467d519a29", default-features = false, features = [ + "kata-cc-native-tls", "verity", ] } [patch.crates-io]