Add migration metrics checking to all test suites

pkg/volume/util/operationexecutor/operation_generator.go

@@ -305,8 +305,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc(
 		}
 
 		// useCSIPlugin will check both CSIMigration and the plugin specific feature gates
-		ucp := useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec)
+		if useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu {
-		if ucp && nu {
 			// The volume represented by this spec is CSI and thus should be migrated
 			attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName)
 			if err != nil || attachableVolumePlugin == nil {
@@ -391,12 +390,15 @@ func (og *operationGenerator) GenerateAttachVolumeFunc(
 	// of the CSI Driver we migrated to. Fixing this requires a larger refactor that
 	// involves determining the plugin_name for the metric generating "CompleteFunc"
 	// during the actual "OperationFunc" and not during this generation function
-	nu, _ := nodeUsingCSIPlugin(og, volumeToAttach.VolumeSpec, volumeToAttach.NodeName)
+
-	ucp := useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec)
+	nu, err := nodeUsingCSIPlugin(og, volumeToAttach.VolumeSpec, volumeToAttach.NodeName)
+	if err != nil {
+		klog.Errorf("GenerateAttachVolumeFunc failed to check if node is using CSI Plugin, metric for this operation may be inaccurate: %v", err)
+	}
 
 	// Need to translate the spec here if the plugin is migrated so that the metrics
 	// emitted show the correct (migrated) plugin
-	if ucp && nu {
+	if useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu {
 		csiSpec, err := translateSpec(volumeToAttach.VolumeSpec)
 		if err == nil {
 			volumeToAttach.VolumeSpec = csiSpec

test/e2e/storage/drivers/in_tree.go

@@ -87,9 +87,9 @@ var _ testsuites.DynamicPVTestDriver = &nfsDriver{}
 func InitNFSDriver() testsuites.TestDriver {
 	return &nfsDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "nfs",
+			Name:             "nfs",
-			PluginName:  "kubernetes.io/nfs",
+			InTreePluginName: "kubernetes.io/nfs",
-			MaxFileSize: testpatterns.FileSizeLarge,
+			MaxFileSize:      testpatterns.FileSizeLarge,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -229,9 +229,9 @@ var _ testsuites.PreprovisionedPVTestDriver = &glusterFSDriver{}
 func InitGlusterFSDriver() testsuites.TestDriver {
 	return &glusterFSDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "gluster",
+			Name:             "gluster",
-			PluginName:  "kubernetes.io/glusterfs",
+			InTreePluginName: "kubernetes.io/glusterfs",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -347,10 +347,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &iSCSIDriver{}
 func InitISCSIDriver() testsuites.TestDriver {
 	return &iSCSIDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "iscsi",
+			Name:             "iscsi",
-			PluginName:  "kubernetes.io/iscsi",
+			InTreePluginName: "kubernetes.io/iscsi",
-			FeatureTag:  "[Feature:Volumes]",
+			FeatureTag:       "[Feature:Volumes]",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext2",
@@ -460,10 +460,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &rbdDriver{}
 func InitRbdDriver() testsuites.TestDriver {
 	return &rbdDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "rbd",
+			Name:             "rbd",
-			PluginName:  "kubernetes.io/rbd",
+			InTreePluginName: "kubernetes.io/rbd",
-			FeatureTag:  "[Feature:Volumes]",
+			FeatureTag:       "[Feature:Volumes]",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext2",
@@ -588,10 +588,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &cephFSDriver{}
 func InitCephFSDriver() testsuites.TestDriver {
 	return &cephFSDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "ceph",
+			Name:             "ceph",
-			PluginName:  "kubernetes.io/cephfs",
+			InTreePluginName: "kubernetes.io/cephfs",
-			FeatureTag:  "[Feature:Volumes]",
+			FeatureTag:       "[Feature:Volumes]",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -688,9 +688,9 @@ var _ testsuites.InlineVolumeTestDriver = &hostPathDriver{}
 func InitHostPathDriver() testsuites.TestDriver {
 	return &hostPathDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "hostPath",
+			Name:             "hostPath",
-			PluginName:  "kubernetes.io/host-path",
+			InTreePluginName: "kubernetes.io/host-path",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -761,9 +761,9 @@ var _ testsuites.InlineVolumeTestDriver = &hostPathSymlinkDriver{}
 func InitHostPathSymlinkDriver() testsuites.TestDriver {
 	return &hostPathSymlinkDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "hostPathSymlink",
+			Name:             "hostPathSymlink",
-			PluginName:  "kubernetes.io/host-path",
+			InTreePluginName: "kubernetes.io/host-path",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -902,9 +902,9 @@ var _ testsuites.InlineVolumeTestDriver = &emptydirDriver{}
 func InitEmptydirDriver() testsuites.TestDriver {
 	return &emptydirDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "emptydir",
+			Name:             "emptydir",
-			PluginName:  "kubernetes.io/empty-dir",
+			InTreePluginName: "kubernetes.io/empty-dir",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
@@ -968,9 +968,9 @@ var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
 func InitCinderDriver() testsuites.TestDriver {
 	return &cinderDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "cinder",
+			Name:             "cinder",
-			PluginName:  "kubernetes.io/cinder",
+			InTreePluginName: "kubernetes.io/cinder",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext3",
@@ -1138,7 +1138,7 @@ func InitGcePdDriver() testsuites.TestDriver {
 	return &gcePdDriver{
 		driverInfo: testsuites.DriverInfo{
 			Name:                 "gcepd",
-			PluginName:           "kubernetes.io/gce-pd",
+			InTreePluginName:     "kubernetes.io/gce-pd",
 			MaxFileSize:          testpatterns.FileSizeMedium,
 			SupportedFsType:      supportedTypes,
 			SupportedMountOption: sets.NewString("debug", "nouid32"),
@@ -1265,9 +1265,9 @@ var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
 func InitVSphereDriver() testsuites.TestDriver {
 	return &vSphereDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "vSphere",
+			Name:             "vSphere",
-			PluginName:  "kubernetes.io/vsphere-volume",
+			InTreePluginName: "kubernetes.io/vsphere-volume",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext4",
@@ -1387,9 +1387,9 @@ var _ testsuites.DynamicPVTestDriver = &azureDriver{}
 func InitAzureDriver() testsuites.TestDriver {
 	return &azureDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "azure",
+			Name:             "azure",
-			PluginName:  "kubernetes.io/azure-file",
+			InTreePluginName: "kubernetes.io/azure-file",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext4",
@@ -1506,9 +1506,9 @@ var _ testsuites.DynamicPVTestDriver = &awsDriver{}
 func InitAwsDriver() testsuites.TestDriver {
 	return &awsDriver{
 		driverInfo: testsuites.DriverInfo{
-			Name:        "aws",
+			Name:             "aws",
-			PluginName:  "kubernetes.io/aws-ebs",
+			InTreePluginName: "kubernetes.io/aws-ebs",
-			MaxFileSize: testpatterns.FileSizeMedium,
+			MaxFileSize:      testpatterns.FileSizeMedium,
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 				"ext3",
@@ -1673,12 +1673,12 @@ func InitLocalDriverWithVolumeType(volumeType utils.LocalVolumeType) func() test
 		}
 		return &localDriver{
 			driverInfo: testsuites.DriverInfo{
-				Name:            "local",
+				Name:             "local",
-				PluginName:      "kubernetes.io/local-volume",
+				InTreePluginName: "kubernetes.io/local-volume",
-				FeatureTag:      featureTag,
+				FeatureTag:       featureTag,
-				MaxFileSize:     maxFileSize,
+				MaxFileSize:      maxFileSize,
-				SupportedFsType: supportedFsTypes,
+				SupportedFsType:  supportedFsTypes,
-				Capabilities:    capabilities,
+				Capabilities:     capabilities,
 			},
 			volumeType: volumeType,
 		}

test/e2e/storage/testsuites/BUILD

@@ -32,7 +32,9 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/csi-translation-lib:go_default_library",
         "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/framework/podlogs:go_default_library",
         "//test/e2e/framework/volume:go_default_library",
         "//test/e2e/storage/testpatterns:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -48,7 +48,7 @@ var (
 )
 
 func init() {
-	migratedPlugins = flag.String("storage.migratedPlugins", "", "comma seperated list of in-tree plugin names of form 'kubernetes.io/{pluginName}' migrated to CSI")
+	migratedPlugins = flag.String("storage.migratedPlugins", "", "comma separated list of in-tree plugin names of form 'kubernetes.io/{pluginName}' migrated to CSI")
 }
 
 type opCounts map[string]int64
@@ -481,31 +481,7 @@ func StartPodLogs(f *framework.Framework) func() {
 	return cancel
 }
 
-func getVolumeOpsFromMetricsForPlugin(ms metrics.ControllerManagerMetrics, pluginName string) opCounts {
+func getVolumeOpsFromMetricsForPlugin(ms metrics.Metrics, pluginName string) opCounts {
-	totOps := opCounts{}
-
-	for method, samples := range ms {
-		switch method {
-		case "storage_operation_status_count":
-			for _, sample := range samples {
-				plugin := string(sample.Metric["volume_plugin"])
-				if pluginName != plugin {
-					continue
-				}
-				opName := string(sample.Metric["operation_name"])
-				if opName == "verify_controller_attached_volume" {
-					// We ignore verify_controller_attached_volume because it does not call into
-					// the plugin. It only watches Node API and updates Actual State of World cache
-					continue
-				}
-				totOps[opName] = totOps[opName] + int64(sample.Value)
-			}
-		}
-	}
-	return totOps
-}
-
-func getVolumeOpsFromKubeletMetricsForPlugin(ms metrics.KubeletMetrics, pluginName string) opCounts {
 	totOps := opCounts{}
 
 	for method, samples := range ms {
@@ -530,6 +506,8 @@ func getVolumeOpsFromKubeletMetricsForPlugin(ms metrics.KubeletMetrics, pluginNa
 }
 
 func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
+	nodeLimit := 25
+
 	metricsGrabber, err := metrics.NewMetricsGrabber(c, nil, true, false, true, false, false)
 
 	if err != nil {
@@ -542,14 +520,22 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
 
 	controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
 	framework.ExpectNoError(err, "Error getting c-m metrics : %v", err)
-	totOps := getVolumeOpsFromMetricsForPlugin(controllerMetrics, pluginName)
+	totOps := getVolumeOpsFromMetricsForPlugin(metrics.Metrics(controllerMetrics), pluginName)
 
+	framework.Logf("Node name not specified for getVolumeOpCounts, falling back to listing nodes from API Server")
 	nodes, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
 	framework.ExpectNoError(err, "Error listing nodes: %v", err)
-	for _, node := range nodes.Items {
+	if len(nodes.Items) <= nodeLimit {
-		nodeMetrics, err := metricsGrabber.GrabFromKubelet(node.GetName())
+		// For large clusters with > nodeLimit nodes it is too time consuming to
-		framework.ExpectNoError(err, "Error getting Kubelet %v metrics: %v", node.GetName(), err)
+		// gather metrics from all nodes. We just ignore the node metrics
-		totOps = addOpCounts(totOps, getVolumeOpsFromKubeletMetricsForPlugin(nodeMetrics, pluginName))
+		// for those clusters
+		for _, node := range nodes.Items {
+			nodeMetrics, err := metricsGrabber.GrabFromKubelet(node.GetName())
+			framework.ExpectNoError(err, "Error getting Kubelet %v metrics: %v", node.GetName(), err)
+			totOps = addOpCounts(totOps, getVolumeOpsFromMetricsForPlugin(metrics.Metrics(nodeMetrics), pluginName))
+		}
+	} else {
+		framework.Logf("Skipping operation metrics gathering from nodes in getVolumeOpCounts, greater than %v nodes", nodeLimit)
 	}
 
 	return totOps
@@ -560,8 +546,7 @@ func addOpCounts(o1 opCounts, o2 opCounts) opCounts {
 	seen := sets.NewString()
 	for op, count := range o1 {
 		seen.Insert(op)
-		totOps[op] = totOps[op] + count
+		totOps[op] = totOps[op] + count + o2[op]
-		totOps[op] = totOps[op] + o2[op]
 	}
 	for op, count := range o2 {
 		if !seen.Has(op) {
@@ -604,33 +589,33 @@ func validateMigrationVolumeOpCounts(cs clientset.Interface, pluginName string,
 		return
 	}
 
-	newInTreeOps, newMigratedOps := getMigrationVolumeOpCounts(cs, pluginName)
-
 	if sets.NewString(strings.Split(*migratedPlugins, ",")...).Has(pluginName) {
 		// If this plugin is migrated based on the test flag storage.migratedPlugins
+		newInTreeOps, _ := getMigrationVolumeOpCounts(cs, pluginName)
+
 		for op, count := range newInTreeOps {
 			if count != oldInTreeOps[op] {
 				framework.Failf("In-tree plugin %v migrated to CSI Driver, however found %v %v metrics for in-tree plugin", pluginName, count-oldInTreeOps[op], op)
 			}
 		}
-		totMigrated := getTotOps(newMigratedOps)
+		// We don't check for migrated metrics because some negative test cases
-		oldTotMigrated := getTotOps(oldMigratedOps)
+		// may not do any volume operations and therefore not emit any metrics
-		if totMigrated-oldTotMigrated <= 0 {
-			framework.Failf("In-tree plugin %v migrated to CSI Driver, however found %v metrics for migrated plugin", pluginName, totMigrated-oldTotMigrated)
-
-		}
 	} else {
 		// In-tree plugin is not migrated
-		totInTree := getTotOps(newInTreeOps)
+		framework.Logf("In-tree plugin %v is not migrated, not validating any metrics", pluginName)
-		oldTotInTree := getTotOps(oldInTreeOps)
-		if totInTree == oldTotInTree {
-			framework.Failf("In-tree plugin %v NOT migrated to CSI Driver, however found did not find any operation metrics for in-tree plugin", pluginName)
-		}
-		// We don't check counts for the Migrated version of the driver because if tests are running in parallel a test could be using
-		// the CSI Driver natively and increase the metrics count
 
-		// TODO(dyzz): Add a dimension to OperationGenerator metrics for "migrated"->true/false so that we can disambiguate migrated metrics
+		// We don't check in-tree plugin metrics because some negative test
-		// and native CSI Driver metrics. This way we can check the counts for migrated version of the driver for stronger negative test case
+		// cases may not do any volume operations and therefore not emit any
+		// metrics
+
+		// We don't check counts for the Migrated version of the driver because
+		// if tests are running in parallel a test could be using the CSI Driver
+		// natively and increase the metrics count
+
+		// TODO(dyzz): Add a dimension to OperationGenerator metrics for
+		// "migrated"->true/false so that we can disambiguate migrated metrics
+		// and native CSI Driver metrics. This way we can check the counts for
+		// migrated version of the driver for stronger negative test case
 		// guarantees (as well as more informative metrics).
 	}
 }

test/e2e/storage/testsuites/multivolume.go

@@ -64,6 +64,9 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
 		ns        *v1.Namespace
 		driver    TestDriver
 		resources []*genericVolumeTestResource
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var (
 		dInfo = driver.GetDriverInfo()
@@ -91,6 +94,7 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
 
 		// Now do the more expensive test initialization.
 		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
 	}
 
 	cleanup := func() {
@@ -102,6 +106,8 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	// This tests below configuration:

test/e2e/storage/testsuites/provisioning.go

@@ -89,6 +89,9 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 		cs       clientset.Interface
 		pvc      *v1.PersistentVolumeClaim
 		sc       *storage.StorageClass
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var (
 		dInfo   = driver.GetDriverInfo()
@@ -119,6 +122,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 
 		// Now do the more expensive test initialization.
 		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
 		l.cs = l.config.Framework.ClientSet
 		claimSize := dDriver.GetClaimSize()
 		l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, pattern.FsType)
@@ -142,6 +146,8 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	It("should provision storage with defaults", func() {

test/e2e/storage/testsuites/subpath.go

@@ -83,6 +83,9 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 		subPathDir        string
 		filePathInSubpath string
 		filePathInVolume  string
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var l local
 
@@ -99,6 +102,7 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 
 		// Now do the more expensive test initialization.
 		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName)
 		l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
 
 		// Setup subPath test dependent resource
@@ -157,6 +161,8 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	It("should support non-existent path", func() {

test/e2e/storage/testsuites/testdriver.go

@@ -128,8 +128,14 @@ const (
 
 // DriverInfo represents static information about a TestDriver.
 type DriverInfo struct {
-	Name       string // Name of the driver, aka the provisioner name.
+	// Internal name of the driver, this is used as a display name in the test
-	FeatureTag string // FeatureTag for the driver
+	// case and test objects
+	Name string
+	// Fully qualified plugin name as registered in Kubernetes of the in-tree
+	// plugin if it exists and is empty if this DriverInfo represents a CSI
+	// Driver
+	InTreePluginName string
+	FeatureTag       string // FeatureTag for the driver
 
 	MaxFileSize          int64               // Max file size to be tested for this driver
 	SupportedFsType      sets.String         // Map of string for supported fs type

test/e2e/storage/testsuites/volume_io.go

@@ -80,6 +80,9 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
 		testCleanup func()
 
 		resource *genericVolumeTestResource
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var (
 		dInfo = driver.GetDriverInfo()
@@ -99,10 +102,13 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
 
 		// Now do the more expensive test initialization.
 		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
+
 		l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
 		if l.resource.volSource == nil {
 			framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
 		}
+
 	}
 
 	cleanup := func() {
@@ -115,6 +121,8 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	It("should write files of various sizes, verify size, validate content [Slow]", func() {

test/e2e/storage/testsuites/volumemode.go

@@ -70,6 +70,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 		ns *v1.Namespace
 		// genericVolumeTestResource contains pv, pvc, sc, etc., owns cleaning that up
 		genericVolumeTestResource
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var (
 		dInfo = driver.GetDriverInfo()
@@ -91,6 +94,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 
 		// Now do the more expensive test initialization.
 		l.config, l.testCleanup = driver.PrepareTest(f)
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
 
 		fsType := pattern.FsType
 		volBindMode := storagev1.VolumeBindingImmediate
@@ -153,6 +157,8 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	// We register different tests depending on the drive

test/e2e/storage/testsuites/volumes.go

@@ -98,6 +98,9 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 		testCleanup func()
 
 		resource *genericVolumeTestResource
+
+		intreeOps   opCounts
+		migratedOps opCounts
 	}
 	var dInfo = driver.GetDriverInfo()
 	var l local
@@ -119,6 +122,8 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 		if l.resource.volSource == nil {
 			framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
 		}
+
+		l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
 	}
 
 	cleanup := func() {
@@ -131,6 +136,8 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 			l.testCleanup()
 			l.testCleanup = nil
 		}
+
+		validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
 	}
 
 	It("should be mountable", func() {