From 726eeb7778f29f4bd68bc7f7e3e3b5d65eeb15c1 Mon Sep 17 00:00:00 2001 From: mowangdk Date: Fri, 10 May 2024 23:14:19 +0800 Subject: [PATCH] chore: Add e2e test for NodeGetVolumeStats --- test/e2e/storage/csimock/base.go | 4 + .../storage/csimock/csi_node_volume_stats.go | 125 ++++++++++++++++++ .../drivers/csi-test/mock/service/node.go | 17 ++- .../drivers/csi-test/mock/service/service.go | 1 + test/e2e/storage/drivers/csi.go | 4 + 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 test/e2e/storage/csimock/csi_node_volume_stats.go diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index 99c69dbf2d0..cfbf2923971 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -55,6 +55,8 @@ const ( csiPodUnschedulableTimeout = 5 * time.Minute csiResizeWaitPeriod = 5 * time.Minute csiVolumeAttachmentTimeout = 7 * time.Minute + // how long to wait for GetVolumeStats + csiNodeVolumeStatWaitPeriod = 1 * time.Minute // how long to wait for Resizing Condition on PVC to appear csiResizingConditionWait = 2 * time.Minute @@ -95,6 +97,7 @@ type testParameters struct { disableResizingOnDriver bool enableSnapshot bool enableVolumeMountGroup bool // enable the VOLUME_MOUNT_GROUP node capability in the CSI mock driver. + enableNodeVolumeStat bool hooks *drivers.Hooks tokenRequests []storagev1.TokenRequest requiresRepublish *bool @@ -164,6 +167,7 @@ func (m *mockDriverSetup) init(ctx context.Context, tp testParameters) { DisableAttach: tp.disableAttach, EnableResizing: tp.enableResizing, EnableNodeExpansion: tp.enableNodeExpansion, + EnableNodeVolumeStat: tp.enableNodeVolumeStat, EnableSnapshot: tp.enableSnapshot, EnableVolumeMountGroup: tp.enableVolumeMountGroup, TokenRequests: tp.tokenRequests, diff --git a/test/e2e/storage/csimock/csi_node_volume_stats.go b/test/e2e/storage/csimock/csi_node_volume_stats.go new file mode 100644 index 00000000000..bfa6e739658 --- /dev/null +++ b/test/e2e/storage/csimock/csi_node_volume_stats.go @@ -0,0 +1,125 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csimock + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" + + "github.com/onsi/ginkgo/v2" +) + +var _ = utils.SIGDescribe("CSI Mock Node Volume Stats", func() { + f := framework.NewDefaultFramework("csi-mock-node-volume-stats") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + f.Context("CSI Mock Node Volume Stats", f.WithSlow(), func() { + trackedCalls := []string{ + "NodeGetVolumeStats", + } + tests := []struct { + name string + expectedCalls []csiCall + nodeVolumeStatRequired bool + nodeGetVolumeStatsHook func(counter int64) error + }{ + { + name: "return abnormal volume stats", + expectedCalls: []csiCall{}, + nodeVolumeStatRequired: false, + nodeGetVolumeStatsHook: func(counter int64) error { + return nil + }, + }, + { + name: "return normal volume stats", + expectedCalls: []csiCall{ + { + expectedMethod: "NodeGetVolumeStats", + expectedError: codes.OK, + }, + }, + nodeVolumeStatRequired: true, + nodeGetVolumeStatsHook: func(counter int64) error { + return nil + }, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + ginkgo.By(fmt.Sprintf("volume stats: %+v", test)) + // Hooks appear to be required for enableNodeVolumeStat. + hooks := createPreHook("NodeGetVolumeStats", test.nodeGetVolumeStatsHook) + m.init(ctx, testParameters{ + registerDriver: true, + enableNodeVolumeStat: test.nodeVolumeStatRequired, + hooks: hooks, + }) + ginkgo.DeferCleanup(m.cleanup) + _, claim, pod := m.createPod(ctx, pvcReference) + if pod == nil { + return + } + // Wait for PVC to get bound to make sure the CSI driver is fully started. + err := e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, f.ClientSet, f.Namespace.Name, claim.Name, time.Second, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err, "while waiting for PVC to get provisioned") + + ginkgo.By("Waiting for pod to be running") + err = e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + ginkgo.By("Waiting for all remaining expected CSI calls") + err = wait.PollUntilContextTimeout(ctx, time.Second, csiNodeVolumeStatWaitPeriod, true, func(c context.Context) (done bool, err error) { + var index int + _, index, err = compareCSICalls(ctx, trackedCalls, test.expectedCalls, m.driver.GetCalls) + if err != nil { + return true, err + } + if index == 0 { + // No CSI call received yet + return false, nil + } + if len(test.expectedCalls) == index { + // all calls received + return true, nil + } + return false, nil + }) + if test.nodeVolumeStatRequired { + framework.ExpectNoError(err, "while waiting for all CSI calls") + } else { + gomega.Expect(err).To(gomega.HaveOccurred(), "an error should have occurred") + } + }) + } + + }) +}) diff --git a/test/e2e/storage/drivers/csi-test/mock/service/node.go b/test/e2e/storage/drivers/csi-test/mock/service/node.go index ce79904aa5e..0c8fb3b2e50 100644 --- a/test/e2e/storage/drivers/csi-test/mock/service/node.go +++ b/test/e2e/storage/drivers/csi-test/mock/service/node.go @@ -352,13 +352,6 @@ func (s *service) NodeGetCapabilities( }, }, }, - { - Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, - }, - }, - }, { Type: &csi.NodeServiceCapability_Rpc{ Rpc: &csi.NodeServiceCapability_RPC{ @@ -377,6 +370,16 @@ func (s *service) NodeGetCapabilities( }) } + if s.config.NodeVolumeStatRequired { + capabilities = append(capabilities, &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, + }, + }, + }) + } + if s.config.VolumeMountGroupRequired { capabilities = append(capabilities, &csi.NodeServiceCapability{ Type: &csi.NodeServiceCapability_Rpc{ diff --git a/test/e2e/storage/drivers/csi-test/mock/service/service.go b/test/e2e/storage/drivers/csi-test/mock/service/service.go index 1c96ddc2982..f30c697db1b 100644 --- a/test/e2e/storage/drivers/csi-test/mock/service/service.go +++ b/test/e2e/storage/drivers/csi-test/mock/service/service.go @@ -55,6 +55,7 @@ type Config struct { DriverName string AttachLimit int64 NodeExpansionRequired bool + NodeVolumeStatRequired bool VolumeMountGroupRequired bool DisableControllerExpansion bool DisableOnlineExpansion bool diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 5aaecc0818b..4d5702e114d 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -306,6 +306,7 @@ type mockCSIDriver struct { requiresRepublish *bool fsGroupPolicy *storagev1.FSGroupPolicy enableVolumeMountGroup bool + enableNodeVolumeStat bool embedded bool calls MockCSICalls embeddedCSIDriver *mockdriver.CSIDriver @@ -354,6 +355,7 @@ type CSIMockDriverOpts struct { EnableNodeExpansion bool EnableSnapshot bool EnableVolumeMountGroup bool + EnableNodeVolumeStat bool TokenRequests []storagev1.TokenRequest RequiresRepublish *bool FSGroupPolicy *storagev1.FSGroupPolicy @@ -507,6 +509,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver { attachable: !driverOpts.DisableAttach, attachLimit: driverOpts.AttachLimit, enableNodeExpansion: driverOpts.EnableNodeExpansion, + enableNodeVolumeStat: driverOpts.EnableNodeVolumeStat, tokenRequests: driverOpts.TokenRequests, requiresRepublish: driverOpts.RequiresRepublish, fsGroupPolicy: driverOpts.FSGroupPolicy, @@ -584,6 +587,7 @@ func (m *mockCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) DriverName: "csi-mock-" + f.UniqueName, AttachLimit: int64(m.attachLimit), NodeExpansionRequired: m.enableNodeExpansion, + NodeVolumeStatRequired: m.enableNodeVolumeStat, VolumeMountGroupRequired: m.enableVolumeMountGroup, EnableTopology: m.enableTopology, IO: proxy.PodDirIO{