From bca1b79728888b307f7ddaf40a60f53c71adbaec Mon Sep 17 00:00:00 2001 From: Cheng Xing Date: Mon, 22 Nov 2021 17:00:39 -0800 Subject: [PATCH] Delegate FSGroup CSI driver e2e: verify fsgroup is passed to CSI calls using mock driver tests --- test/e2e/storage/csi_mock_volume.go | 100 +++++++++++++++--- .../drivers/csi-test/mock/service/node.go | 10 ++ .../drivers/csi-test/mock/service/service.go | 1 + test/e2e/storage/drivers/csi.go | 92 ++++++++-------- 4 files changed, 147 insertions(+), 56 deletions(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index aba74f9d7dd..5e7834f0dc4 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -27,6 +27,7 @@ import ( "sync/atomic" "time" + csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" @@ -107,6 +108,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { // just disable resizing on driver it overrides enableResizing flag for CSI mock driver disableResizingOnDriver bool enableSnapshot bool + enableVolumeMountGroup bool // enable the VOLUME_MOUNT_GROUP node capability in the CSI mock driver. hooks *drivers.Hooks tokenRequests []storagev1.TokenRequest requiresRepublish *bool @@ -140,18 +142,19 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { cs := f.ClientSet var err error driverOpts := drivers.CSIMockDriverOpts{ - RegisterDriver: tp.registerDriver, - PodInfo: tp.podInfo, - StorageCapacity: tp.storageCapacity, - EnableTopology: tp.enableTopology, - AttachLimit: tp.attachLimit, - DisableAttach: tp.disableAttach, - EnableResizing: tp.enableResizing, - EnableNodeExpansion: tp.enableNodeExpansion, - EnableSnapshot: tp.enableSnapshot, - TokenRequests: tp.tokenRequests, - RequiresRepublish: tp.requiresRepublish, - FSGroupPolicy: tp.fsGroupPolicy, + RegisterDriver: tp.registerDriver, + PodInfo: tp.podInfo, + StorageCapacity: tp.storageCapacity, + EnableTopology: tp.enableTopology, + AttachLimit: tp.attachLimit, + DisableAttach: tp.disableAttach, + EnableResizing: tp.enableResizing, + EnableNodeExpansion: tp.enableNodeExpansion, + EnableSnapshot: tp.enableSnapshot, + EnableVolumeMountGroup: tp.enableVolumeMountGroup, + TokenRequests: tp.tokenRequests, + RequiresRepublish: tp.requiresRepublish, + FSGroupPolicy: tp.fsGroupPolicy, } // At the moment, only tests which need hooks are @@ -1715,6 +1718,55 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { } }) + ginkgo.Context("Delegate FSGroup to CSI driver [LinuxOnly]", func() { + tests := []struct { + name string + enableVolumeMountGroup bool + }{ + { + name: "should pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", + enableVolumeMountGroup: true, + }, + { + name: "should not pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", + enableVolumeMountGroup: false, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func() { + var nodeStageFsGroup, nodePublishFsGroup string + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("FSGroupPolicy is only applied on linux nodes -- skipping") + } + init(testParameters{ + disableAttach: true, + registerDriver: true, + enableVolumeMountGroup: t.enableVolumeMountGroup, + hooks: createFSGroupRequestPreHook(&nodeStageFsGroup, &nodePublishFsGroup), + }) + defer cleanup() + + fsGroupVal := int64(rand.Int63n(20000) + 1024) + fsGroup := &fsGroupVal + fsGroupStr := strconv.FormatInt(fsGroupVal, 10 /* base */) + + _, _, pod := createPodWithFSGroup(fsGroup) /* persistent volume */ + + err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "failed to start pod") + + if t.enableVolumeMountGroup { + framework.ExpectEqual(nodeStageFsGroup, fsGroupStr, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodeStageFsGroup) + framework.ExpectEqual(nodePublishFsGroup, fsGroupStr, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodePublishFsGroup) + } else { + framework.ExpectEmpty(nodeStageFsGroup, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to be empty; got: %q", nodeStageFsGroup) + framework.ExpectEmpty(nodePublishFsGroup, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to empty; got: %q", nodePublishFsGroup) + } + }) + } + }) + ginkgo.Context("CSI Volume Snapshots secrets [Feature:VolumeSnapshotDataSource]", func() { var ( @@ -2460,6 +2512,30 @@ func createPreHook(method string, callback func(counter int64) error) *drivers.H } } +// createFSGroupRequestPreHook creates a hook that records the fsGroup passed in +// through NodeStageVolume and NodePublishVolume calls. +func createFSGroupRequestPreHook(nodeStageFsGroup, nodePublishFsGroup *string) *drivers.Hooks { + return &drivers.Hooks{ + Pre: func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { + nodeStageRequest, ok := request.(csipbv1.NodeStageVolumeRequest) + if ok { + mountVolume := nodeStageRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodeStageFsGroup = mountVolume.VolumeMountGroup + } + } + nodePublishRequest, ok := request.(csipbv1.NodePublishVolumeRequest) + if ok { + mountVolume := nodePublishRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodePublishFsGroup = mountVolume.VolumeMountGroup + } + } + return nil, nil + }, + } +} + type snapshotMetricsTestConfig struct { // expected values metricName string diff --git a/test/e2e/storage/drivers/csi-test/mock/service/node.go b/test/e2e/storage/drivers/csi-test/mock/service/node.go index ddffea048af..ce79904aa5e 100644 --- a/test/e2e/storage/drivers/csi-test/mock/service/node.go +++ b/test/e2e/storage/drivers/csi-test/mock/service/node.go @@ -377,6 +377,16 @@ func (s *service) NodeGetCapabilities( }) } + if s.config.VolumeMountGroupRequired { + capabilities = append(capabilities, &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP, + }, + }, + }) + } + return &csi.NodeGetCapabilitiesResponse{ Capabilities: capabilities, }, nil diff --git a/test/e2e/storage/drivers/csi-test/mock/service/service.go b/test/e2e/storage/drivers/csi-test/mock/service/service.go index 45c7493a6f5..03146c1d1df 100644 --- a/test/e2e/storage/drivers/csi-test/mock/service/service.go +++ b/test/e2e/storage/drivers/csi-test/mock/service/service.go @@ -55,6 +55,7 @@ type Config struct { DriverName string AttachLimit int64 NodeExpansionRequired bool + VolumeMountGroupRequired bool DisableControllerExpansion bool DisableOnlineExpansion bool PermissiveTargetPath bool diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 021e6200f28..c387c7e55e1 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -299,21 +299,22 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo // mockCSI type mockCSIDriver struct { - driverInfo storageframework.DriverInfo - manifests []string - podInfo *bool - storageCapacity *bool - attachable bool - attachLimit int - enableTopology bool - enableNodeExpansion bool - hooks Hooks - tokenRequests []storagev1.TokenRequest - requiresRepublish *bool - fsGroupPolicy *storagev1.FSGroupPolicy - embedded bool - calls MockCSICalls - embeddedCSIDriver *mockdriver.CSIDriver + driverInfo storageframework.DriverInfo + manifests []string + podInfo *bool + storageCapacity *bool + attachable bool + attachLimit int + enableTopology bool + enableNodeExpansion bool + hooks Hooks + tokenRequests []storagev1.TokenRequest + requiresRepublish *bool + fsGroupPolicy *storagev1.FSGroupPolicy + enableVolumeMountGroup bool + embedded bool + calls MockCSICalls + embeddedCSIDriver *mockdriver.CSIDriver // Additional values set during PrepareTest clientSet kubernetes.Interface @@ -347,18 +348,19 @@ type MockCSITestDriver interface { // CSIMockDriverOpts defines options used for csi driver type CSIMockDriverOpts struct { - RegisterDriver bool - DisableAttach bool - PodInfo *bool - StorageCapacity *bool - AttachLimit int - EnableTopology bool - EnableResizing bool - EnableNodeExpansion bool - EnableSnapshot bool - TokenRequests []storagev1.TokenRequest - RequiresRepublish *bool - FSGroupPolicy *storagev1.FSGroupPolicy + RegisterDriver bool + DisableAttach bool + PodInfo *bool + StorageCapacity *bool + AttachLimit int + EnableTopology bool + EnableResizing bool + EnableNodeExpansion bool + EnableSnapshot bool + EnableVolumeMountGroup bool + TokenRequests []storagev1.TokenRequest + RequiresRepublish *bool + FSGroupPolicy *storagev1.FSGroupPolicy // Embedded defines whether the CSI mock driver runs // inside the cluster (false, the default) or just a proxy @@ -499,18 +501,19 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver { storageframework.CapVolumeLimits: true, }, }, - manifests: driverManifests, - podInfo: driverOpts.PodInfo, - storageCapacity: driverOpts.StorageCapacity, - enableTopology: driverOpts.EnableTopology, - attachable: !driverOpts.DisableAttach, - attachLimit: driverOpts.AttachLimit, - enableNodeExpansion: driverOpts.EnableNodeExpansion, - tokenRequests: driverOpts.TokenRequests, - requiresRepublish: driverOpts.RequiresRepublish, - fsGroupPolicy: driverOpts.FSGroupPolicy, - embedded: driverOpts.Embedded, - hooks: driverOpts.Hooks, + manifests: driverManifests, + podInfo: driverOpts.PodInfo, + storageCapacity: driverOpts.StorageCapacity, + enableTopology: driverOpts.EnableTopology, + attachable: !driverOpts.DisableAttach, + attachLimit: driverOpts.AttachLimit, + enableNodeExpansion: driverOpts.EnableNodeExpansion, + tokenRequests: driverOpts.TokenRequests, + requiresRepublish: driverOpts.RequiresRepublish, + fsGroupPolicy: driverOpts.FSGroupPolicy, + enableVolumeMountGroup: driverOpts.EnableVolumeMountGroup, + embedded: driverOpts.Embedded, + hooks: driverOpts.Hooks, } } @@ -573,11 +576,12 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.P containername := "mock" ctx, cancel := context.WithCancel(context.Background()) serviceConfig := mockservice.Config{ - DisableAttach: !m.attachable, - DriverName: "csi-mock-" + f.UniqueName, - AttachLimit: int64(m.attachLimit), - NodeExpansionRequired: m.enableNodeExpansion, - EnableTopology: m.enableTopology, + DisableAttach: !m.attachable, + DriverName: "csi-mock-" + f.UniqueName, + AttachLimit: int64(m.attachLimit), + NodeExpansionRequired: m.enableNodeExpansion, + VolumeMountGroupRequired: m.enableVolumeMountGroup, + EnableTopology: m.enableTopology, IO: proxy.PodDirIO{ F: f, Namespace: m.driverNamespace.Name,