diff --git a/test/integration/scheduler/queueing/queue.go b/test/integration/scheduler/queueing/queue.go index f1ee6dfa795..581a13e268d 100644 --- a/test/integration/scheduler/queueing/queue.go +++ b/test/integration/scheduler/queueing/queue.go @@ -66,6 +66,8 @@ type CoreResourceEnqueueTestCase struct { InitialCSIDrivers []*storagev1.CSIDriver // InitialStorageCapacities are the list of CSIStorageCapacity to be created at first. InitialStorageCapacities []*storagev1.CSIStorageCapacity + // InitialVolumeAttachment is the list of VolumeAttachment to be created at first. + InitialVolumeAttachment []*storagev1.VolumeAttachment // Pods are the list of Pods to be created. // All of them are expected to be unschedulable at first. Pods []*v1.Pod @@ -1977,7 +1979,7 @@ var CoreResourceEnqueueTestCases = []*CoreResourceEnqueueTestCase{ EnableSchedulingQueueHint: sets.New(true), }, { - Name: "Pod rejected the CSI plugin is requeued when the CSINode is added", + Name: "Pod rejected the CSI NodeVolumeLimits plugin is requeued when the CSINode is added", InitialNodes: []*v1.Node{st.MakeNode().Name("fake-node").Label("node", "fake-node").Obj()}, InitialPVs: []*v1.PersistentVolume{ st.MakePersistentVolume().Name("pv1"). @@ -2016,7 +2018,7 @@ var CoreResourceEnqueueTestCases = []*CoreResourceEnqueueTestCase{ EnableSchedulingQueueHint: sets.New(true), }, { - Name: "Pod rejected with PVC by the CSI plugin is requeued when the pod having related PVC is deleted", + Name: "Pod rejected with PVC by the CSI NodeVolumeLimits plugin is requeued when the pod having related PVC is deleted", InitialNodes: []*v1.Node{st.MakeNode().Name("fake-node").Label("node", "fake-node").Obj()}, InitialPVs: []*v1.PersistentVolume{ st.MakePersistentVolume().Name("pv1"). @@ -2069,7 +2071,7 @@ var CoreResourceEnqueueTestCases = []*CoreResourceEnqueueTestCase{ WantRequeuedPods: sets.New("pod2"), }, { - Name: "Pod rejected with PVC by the CSI plugin is requeued when the related PVC is added", + Name: "Pod rejected with PVC by the CSI NodeVolumeLimits plugin is requeued when the related PVC is added", InitialNodes: []*v1.Node{st.MakeNode().Name("fake-node").Label("node", "fake-node").Obj()}, EnablePlugins: []string{"VolumeBinding"}, InitialPVs: []*v1.PersistentVolume{ @@ -2123,6 +2125,138 @@ var CoreResourceEnqueueTestCases = []*CoreResourceEnqueueTestCase{ WantRequeuedPods: sets.New("pod2"), EnableSchedulingQueueHint: sets.New(true), }, + { + Name: "Pod with PVC rejected the CSI NodeVolumeLimits plugin is requeued when the VolumeAttachment is deleted", + InitialNodes: []*v1.Node{st.MakeNode().Name("fake-node").Label("node", "fake-node").Obj()}, + InitialPVs: []*v1.PersistentVolume{ + st.MakePersistentVolume().Name("pv1"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Capacity(v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}). + PersistentVolumeSource(v1.PersistentVolumeSource{CSI: &v1.CSIPersistentVolumeSource{Driver: "csidriver", VolumeHandle: "volumehandle"}}). + Obj()}, + InitialPVCs: []*v1.PersistentVolumeClaim{ + st.MakePersistentVolumeClaim(). + Name("pvc1"). + Annotation(volume.AnnBindCompleted, "true"). + VolumeName("pv1"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Resources(v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}}). + Obj(), + }, + InitialVolumeAttachment: []*storagev1.VolumeAttachment{ + st.MakeVolumeAttachment().Name("volumeattachment1"). + NodeName("fake-node"). + Attacher("test.storage.gke.io"). + Source(storagev1.VolumeAttachmentSource{PersistentVolumeName: ptr.To("pv1")}). + Attached(true). + Obj(), + }, + InitialCSIDrivers: []*storagev1.CSIDriver{ + st.MakeCSIDriver().Name("csidriver").StorageCapacity(ptr.To(true)).Obj(), + }, + InitialCSINodes: []*storagev1.CSINode{ + st.MakeCSINode().Name("fake-node").Driver(storagev1.CSINodeDriver{Name: "csidriver", NodeID: "fake-node", Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(0)), + }}).Obj(), + }, + Pods: []*v1.Pod{ + st.MakePod().Name("pod1").Container("image").PVC("pvc1").Obj(), + }, + TriggerFn: func(testCtx *testutils.TestContext) (map[framework.ClusterEvent]uint64, error) { + if err := testCtx.ClientSet.StorageV1().VolumeAttachments().Delete(testCtx.Ctx, "volumeattachment1", metav1.DeleteOptions{}); err != nil { + return nil, fmt.Errorf("failed to delete VolumeAttachment: %w", err) + } + return map[framework.ClusterEvent]uint64{{Resource: framework.VolumeAttachment, ActionType: framework.Delete}: 1}, nil + }, + WantRequeuedPods: sets.New("pod1"), + EnableSchedulingQueueHint: sets.New(true), + }, + { + Name: "Pod with Inline Migratable volume (AWSEBSDriver and GCEPDDriver) rejected the CSI NodeVolumeLimits plugin is requeued (only AWSEBSDriver) when the VolumeAttachment is deleted", + InitialNodes: []*v1.Node{ + st.MakeNode().Name("fake-node").Label("node", "fake-node").Obj(), + }, + InitialPVs: []*v1.PersistentVolume{ + st.MakePersistentVolume().Name("pv1"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Capacity(v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}). + PersistentVolumeSource(v1.PersistentVolumeSource{CSI: &v1.CSIPersistentVolumeSource{Driver: "ebs.csi.aws.com", VolumeHandle: "volumehandle"}}). + Obj(), + st.MakePersistentVolume().Name("pv2"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Capacity(v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}). + PersistentVolumeSource(v1.PersistentVolumeSource{CSI: &v1.CSIPersistentVolumeSource{Driver: "pd.csi.storage.gke.io", VolumeHandle: "volumehandle"}}). + Obj(), + }, + InitialPVCs: []*v1.PersistentVolumeClaim{ + st.MakePersistentVolumeClaim(). + Name("pvc1"). + Annotation(volume.AnnBindCompleted, "true"). + VolumeName("pv1"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Resources(v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}}). + Obj(), + st.MakePersistentVolumeClaim(). + Name("pvc2"). + Annotation(volume.AnnBindCompleted, "true"). + VolumeName("pv2"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}). + Resources(v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}}). + Obj(), + }, + InitialVolumeAttachment: []*storagev1.VolumeAttachment{ + st.MakeVolumeAttachment().Name("volumeattachment1"). + NodeName("fake-node"). + Attacher("ebs.csi.aws.com"). + Source(storagev1.VolumeAttachmentSource{PersistentVolumeName: ptr.To("pv1")}). + Attached(true). + Obj(), + st.MakeVolumeAttachment().Name("volumeattachment2"). + NodeName("fake-node"). + Attacher("pd.csi.storage.gke.io"). + Source(storagev1.VolumeAttachmentSource{PersistentVolumeName: ptr.To("pv2")}). + Attached(true). + Obj(), + }, + InitialCSINodes: []*storagev1.CSINode{ + st.MakeCSINode().Name("fake-node"). + Driver(storagev1.CSINodeDriver{Name: "ebs.csi.aws.com", NodeID: "fake-node", Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(0))}}). + Driver(storagev1.CSINodeDriver{Name: "pd.csi.storage.gke.io", NodeID: "fake-node", Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.To(int32(0))}}). + Obj(), + }, + Pods: []*v1.Pod{ + st.MakePod().Name("pod1").Container("image").Volume( + v1.Volume{ + Name: "vol1", + VolumeSource: v1.VolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "pv1", + }, + }, + }, + ).Obj(), + st.MakePod().Name("pod2").Container("image").Volume( + v1.Volume{ + Name: "vol2", + VolumeSource: v1.VolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "pv2", + }, + }, + }, + ).Obj(), + }, + TriggerFn: func(testCtx *testutils.TestContext) (map[framework.ClusterEvent]uint64, error) { + if err := testCtx.ClientSet.StorageV1().VolumeAttachments().Delete(testCtx.Ctx, "volumeattachment1", metav1.DeleteOptions{}); err != nil { + return nil, fmt.Errorf("failed to delete VolumeAttachment: %w", err) + } + return map[framework.ClusterEvent]uint64{{Resource: framework.VolumeAttachment, ActionType: framework.Delete}: 1}, nil + }, + WantRequeuedPods: sets.New("pod1"), + EnableSchedulingQueueHint: sets.New(true), + }, } // TestCoreResourceEnqueue verify Pods failed by in-tree default plugins can be @@ -2191,6 +2325,12 @@ func RunTestCoreResourceEnqueue(t *testing.T, tt *CoreResourceEnqueueTestCase) { } } + for _, va := range tt.InitialVolumeAttachment { + if _, err := cs.StorageV1().VolumeAttachments().Create(ctx, va, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create a VolumeAttachment %q: %v", va.Name, err) + } + } + for _, csidriver := range tt.InitialCSIDrivers { if _, err := cs.StorageV1().CSIDrivers().Create(ctx, csidriver, metav1.CreateOptions{}); err != nil { t.Fatalf("Failed to create a CSIDriver %q: %v", csidriver.Name, err)