Add migration metrics checking to all test suites

This commit is contained in:
David Zhu 2019-04-25 16:20:19 -07:00
parent 213cc99ead
commit 1271237d23
11 changed files with 137 additions and 103 deletions

View File

@ -305,8 +305,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc(
}
// useCSIPlugin will check both CSIMigration and the plugin specific feature gates
ucp := useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec)
if ucp && nu {
if useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu {
// The volume represented by this spec is CSI and thus should be migrated
attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName)
if err != nil || attachableVolumePlugin == nil {
@ -391,12 +390,15 @@ func (og *operationGenerator) GenerateAttachVolumeFunc(
// of the CSI Driver we migrated to. Fixing this requires a larger refactor that
// involves determining the plugin_name for the metric generating "CompleteFunc"
// during the actual "OperationFunc" and not during this generation function
nu, _ := nodeUsingCSIPlugin(og, volumeToAttach.VolumeSpec, volumeToAttach.NodeName)
ucp := useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec)
nu, err := nodeUsingCSIPlugin(og, volumeToAttach.VolumeSpec, volumeToAttach.NodeName)
if err != nil {
klog.Errorf("GenerateAttachVolumeFunc failed to check if node is using CSI Plugin, metric for this operation may be inaccurate: %v", err)
}
// Need to translate the spec here if the plugin is migrated so that the metrics
// emitted show the correct (migrated) plugin
if ucp && nu {
if useCSIPlugin(og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu {
csiSpec, err := translateSpec(volumeToAttach.VolumeSpec)
if err == nil {
volumeToAttach.VolumeSpec = csiSpec

View File

@ -87,9 +87,9 @@ var _ testsuites.DynamicPVTestDriver = &nfsDriver{}
func InitNFSDriver() testsuites.TestDriver {
return &nfsDriver{
driverInfo: testsuites.DriverInfo{
Name: "nfs",
PluginName: "kubernetes.io/nfs",
MaxFileSize: testpatterns.FileSizeLarge,
Name: "nfs",
InTreePluginName: "kubernetes.io/nfs",
MaxFileSize: testpatterns.FileSizeLarge,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -229,9 +229,9 @@ var _ testsuites.PreprovisionedPVTestDriver = &glusterFSDriver{}
func InitGlusterFSDriver() testsuites.TestDriver {
return &glusterFSDriver{
driverInfo: testsuites.DriverInfo{
Name: "gluster",
PluginName: "kubernetes.io/glusterfs",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "gluster",
InTreePluginName: "kubernetes.io/glusterfs",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -347,10 +347,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &iSCSIDriver{}
func InitISCSIDriver() testsuites.TestDriver {
return &iSCSIDriver{
driverInfo: testsuites.DriverInfo{
Name: "iscsi",
PluginName: "kubernetes.io/iscsi",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "iscsi",
InTreePluginName: "kubernetes.io/iscsi",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext2",
@ -460,10 +460,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &rbdDriver{}
func InitRbdDriver() testsuites.TestDriver {
return &rbdDriver{
driverInfo: testsuites.DriverInfo{
Name: "rbd",
PluginName: "kubernetes.io/rbd",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "rbd",
InTreePluginName: "kubernetes.io/rbd",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext2",
@ -588,10 +588,10 @@ var _ testsuites.PreprovisionedPVTestDriver = &cephFSDriver{}
func InitCephFSDriver() testsuites.TestDriver {
return &cephFSDriver{
driverInfo: testsuites.DriverInfo{
Name: "ceph",
PluginName: "kubernetes.io/cephfs",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "ceph",
InTreePluginName: "kubernetes.io/cephfs",
FeatureTag: "[Feature:Volumes]",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -688,9 +688,9 @@ var _ testsuites.InlineVolumeTestDriver = &hostPathDriver{}
func InitHostPathDriver() testsuites.TestDriver {
return &hostPathDriver{
driverInfo: testsuites.DriverInfo{
Name: "hostPath",
PluginName: "kubernetes.io/host-path",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "hostPath",
InTreePluginName: "kubernetes.io/host-path",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -761,9 +761,9 @@ var _ testsuites.InlineVolumeTestDriver = &hostPathSymlinkDriver{}
func InitHostPathSymlinkDriver() testsuites.TestDriver {
return &hostPathSymlinkDriver{
driverInfo: testsuites.DriverInfo{
Name: "hostPathSymlink",
PluginName: "kubernetes.io/host-path",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "hostPathSymlink",
InTreePluginName: "kubernetes.io/host-path",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -902,9 +902,9 @@ var _ testsuites.InlineVolumeTestDriver = &emptydirDriver{}
func InitEmptydirDriver() testsuites.TestDriver {
return &emptydirDriver{
driverInfo: testsuites.DriverInfo{
Name: "emptydir",
PluginName: "kubernetes.io/empty-dir",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "emptydir",
InTreePluginName: "kubernetes.io/empty-dir",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
@ -968,9 +968,9 @@ var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
func InitCinderDriver() testsuites.TestDriver {
return &cinderDriver{
driverInfo: testsuites.DriverInfo{
Name: "cinder",
PluginName: "kubernetes.io/cinder",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "cinder",
InTreePluginName: "kubernetes.io/cinder",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext3",
@ -1138,7 +1138,7 @@ func InitGcePdDriver() testsuites.TestDriver {
return &gcePdDriver{
driverInfo: testsuites.DriverInfo{
Name: "gcepd",
PluginName: "kubernetes.io/gce-pd",
InTreePluginName: "kubernetes.io/gce-pd",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: supportedTypes,
SupportedMountOption: sets.NewString("debug", "nouid32"),
@ -1265,9 +1265,9 @@ var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
func InitVSphereDriver() testsuites.TestDriver {
return &vSphereDriver{
driverInfo: testsuites.DriverInfo{
Name: "vSphere",
PluginName: "kubernetes.io/vsphere-volume",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "vSphere",
InTreePluginName: "kubernetes.io/vsphere-volume",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext4",
@ -1387,9 +1387,9 @@ var _ testsuites.DynamicPVTestDriver = &azureDriver{}
func InitAzureDriver() testsuites.TestDriver {
return &azureDriver{
driverInfo: testsuites.DriverInfo{
Name: "azure",
PluginName: "kubernetes.io/azure-file",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "azure",
InTreePluginName: "kubernetes.io/azure-file",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext4",
@ -1506,9 +1506,9 @@ var _ testsuites.DynamicPVTestDriver = &awsDriver{}
func InitAwsDriver() testsuites.TestDriver {
return &awsDriver{
driverInfo: testsuites.DriverInfo{
Name: "aws",
PluginName: "kubernetes.io/aws-ebs",
MaxFileSize: testpatterns.FileSizeMedium,
Name: "aws",
InTreePluginName: "kubernetes.io/aws-ebs",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext3",
@ -1673,12 +1673,12 @@ func InitLocalDriverWithVolumeType(volumeType utils.LocalVolumeType) func() test
}
return &localDriver{
driverInfo: testsuites.DriverInfo{
Name: "local",
PluginName: "kubernetes.io/local-volume",
FeatureTag: featureTag,
MaxFileSize: maxFileSize,
SupportedFsType: supportedFsTypes,
Capabilities: capabilities,
Name: "local",
InTreePluginName: "kubernetes.io/local-volume",
FeatureTag: featureTag,
MaxFileSize: maxFileSize,
SupportedFsType: supportedFsTypes,
Capabilities: capabilities,
},
volumeType: volumeType,
}

View File

@ -32,7 +32,9 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/csi-translation-lib:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
"//test/e2e/framework/podlogs:go_default_library",
"//test/e2e/framework/volume:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library",

View File

@ -48,7 +48,7 @@ var (
)
func init() {
migratedPlugins = flag.String("storage.migratedPlugins", "", "comma seperated list of in-tree plugin names of form 'kubernetes.io/{pluginName}' migrated to CSI")
migratedPlugins = flag.String("storage.migratedPlugins", "", "comma separated list of in-tree plugin names of form 'kubernetes.io/{pluginName}' migrated to CSI")
}
type opCounts map[string]int64
@ -481,31 +481,7 @@ func StartPodLogs(f *framework.Framework) func() {
return cancel
}
func getVolumeOpsFromMetricsForPlugin(ms metrics.ControllerManagerMetrics, pluginName string) opCounts {
totOps := opCounts{}
for method, samples := range ms {
switch method {
case "storage_operation_status_count":
for _, sample := range samples {
plugin := string(sample.Metric["volume_plugin"])
if pluginName != plugin {
continue
}
opName := string(sample.Metric["operation_name"])
if opName == "verify_controller_attached_volume" {
// We ignore verify_controller_attached_volume because it does not call into
// the plugin. It only watches Node API and updates Actual State of World cache
continue
}
totOps[opName] = totOps[opName] + int64(sample.Value)
}
}
}
return totOps
}
func getVolumeOpsFromKubeletMetricsForPlugin(ms metrics.KubeletMetrics, pluginName string) opCounts {
func getVolumeOpsFromMetricsForPlugin(ms metrics.Metrics, pluginName string) opCounts {
totOps := opCounts{}
for method, samples := range ms {
@ -530,6 +506,8 @@ func getVolumeOpsFromKubeletMetricsForPlugin(ms metrics.KubeletMetrics, pluginNa
}
func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
nodeLimit := 25
metricsGrabber, err := metrics.NewMetricsGrabber(c, nil, true, false, true, false, false)
if err != nil {
@ -542,14 +520,22 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts {
controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
framework.ExpectNoError(err, "Error getting c-m metrics : %v", err)
totOps := getVolumeOpsFromMetricsForPlugin(controllerMetrics, pluginName)
totOps := getVolumeOpsFromMetricsForPlugin(metrics.Metrics(controllerMetrics), pluginName)
framework.Logf("Node name not specified for getVolumeOpCounts, falling back to listing nodes from API Server")
nodes, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
framework.ExpectNoError(err, "Error listing nodes: %v", err)
for _, node := range nodes.Items {
nodeMetrics, err := metricsGrabber.GrabFromKubelet(node.GetName())
framework.ExpectNoError(err, "Error getting Kubelet %v metrics: %v", node.GetName(), err)
totOps = addOpCounts(totOps, getVolumeOpsFromKubeletMetricsForPlugin(nodeMetrics, pluginName))
if len(nodes.Items) <= nodeLimit {
// For large clusters with > nodeLimit nodes it is too time consuming to
// gather metrics from all nodes. We just ignore the node metrics
// for those clusters
for _, node := range nodes.Items {
nodeMetrics, err := metricsGrabber.GrabFromKubelet(node.GetName())
framework.ExpectNoError(err, "Error getting Kubelet %v metrics: %v", node.GetName(), err)
totOps = addOpCounts(totOps, getVolumeOpsFromMetricsForPlugin(metrics.Metrics(nodeMetrics), pluginName))
}
} else {
framework.Logf("Skipping operation metrics gathering from nodes in getVolumeOpCounts, greater than %v nodes", nodeLimit)
}
return totOps
@ -560,8 +546,7 @@ func addOpCounts(o1 opCounts, o2 opCounts) opCounts {
seen := sets.NewString()
for op, count := range o1 {
seen.Insert(op)
totOps[op] = totOps[op] + count
totOps[op] = totOps[op] + o2[op]
totOps[op] = totOps[op] + count + o2[op]
}
for op, count := range o2 {
if !seen.Has(op) {
@ -604,33 +589,33 @@ func validateMigrationVolumeOpCounts(cs clientset.Interface, pluginName string,
return
}
newInTreeOps, newMigratedOps := getMigrationVolumeOpCounts(cs, pluginName)
if sets.NewString(strings.Split(*migratedPlugins, ",")...).Has(pluginName) {
// If this plugin is migrated based on the test flag storage.migratedPlugins
newInTreeOps, _ := getMigrationVolumeOpCounts(cs, pluginName)
for op, count := range newInTreeOps {
if count != oldInTreeOps[op] {
framework.Failf("In-tree plugin %v migrated to CSI Driver, however found %v %v metrics for in-tree plugin", pluginName, count-oldInTreeOps[op], op)
}
}
totMigrated := getTotOps(newMigratedOps)
oldTotMigrated := getTotOps(oldMigratedOps)
if totMigrated-oldTotMigrated <= 0 {
framework.Failf("In-tree plugin %v migrated to CSI Driver, however found %v metrics for migrated plugin", pluginName, totMigrated-oldTotMigrated)
}
// We don't check for migrated metrics because some negative test cases
// may not do any volume operations and therefore not emit any metrics
} else {
// In-tree plugin is not migrated
totInTree := getTotOps(newInTreeOps)
oldTotInTree := getTotOps(oldInTreeOps)
if totInTree == oldTotInTree {
framework.Failf("In-tree plugin %v NOT migrated to CSI Driver, however found did not find any operation metrics for in-tree plugin", pluginName)
}
// We don't check counts for the Migrated version of the driver because if tests are running in parallel a test could be using
// the CSI Driver natively and increase the metrics count
framework.Logf("In-tree plugin %v is not migrated, not validating any metrics", pluginName)
// TODO(dyzz): Add a dimension to OperationGenerator metrics for "migrated"->true/false so that we can disambiguate migrated metrics
// and native CSI Driver metrics. This way we can check the counts for migrated version of the driver for stronger negative test case
// We don't check in-tree plugin metrics because some negative test
// cases may not do any volume operations and therefore not emit any
// metrics
// We don't check counts for the Migrated version of the driver because
// if tests are running in parallel a test could be using the CSI Driver
// natively and increase the metrics count
// TODO(dyzz): Add a dimension to OperationGenerator metrics for
// "migrated"->true/false so that we can disambiguate migrated metrics
// and native CSI Driver metrics. This way we can check the counts for
// migrated version of the driver for stronger negative test case
// guarantees (as well as more informative metrics).
}
}

View File

@ -64,6 +64,9 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
ns *v1.Namespace
driver TestDriver
resources []*genericVolumeTestResource
intreeOps opCounts
migratedOps opCounts
}
var (
dInfo = driver.GetDriverInfo()
@ -91,6 +94,7 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
}
cleanup := func() {
@ -102,6 +106,8 @@ func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatter
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
}
// This tests below configuration:

View File

@ -89,6 +89,9 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
cs clientset.Interface
pvc *v1.PersistentVolumeClaim
sc *storage.StorageClass
intreeOps opCounts
migratedOps opCounts
}
var (
dInfo = driver.GetDriverInfo()
@ -119,6 +122,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
l.cs = l.config.Framework.ClientSet
claimSize := dDriver.GetClaimSize()
l.sc = dDriver.GetDynamicProvisionStorageClass(l.config, pattern.FsType)
@ -142,6 +146,8 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
}
It("should provision storage with defaults", func() {

View File

@ -83,6 +83,9 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
subPathDir string
filePathInSubpath string
filePathInVolume string
intreeOps opCounts
migratedOps opCounts
}
var l local
@ -99,6 +102,7 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName)
l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
// Setup subPath test dependent resource
@ -157,6 +161,8 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
}
It("should support non-existent path", func() {

View File

@ -128,8 +128,14 @@ const (
// DriverInfo represents static information about a TestDriver.
type DriverInfo struct {
Name string // Name of the driver, aka the provisioner name.
FeatureTag string // FeatureTag for the driver
// Internal name of the driver, this is used as a display name in the test
// case and test objects
Name string
// Fully qualified plugin name as registered in Kubernetes of the in-tree
// plugin if it exists and is empty if this DriverInfo represents a CSI
// Driver
InTreePluginName string
FeatureTag string // FeatureTag for the driver
MaxFileSize int64 // Max file size to be tested for this driver
SupportedFsType sets.String // Map of string for supported fs type

View File

@ -80,6 +80,9 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
testCleanup func()
resource *genericVolumeTestResource
intreeOps opCounts
migratedOps opCounts
}
var (
dInfo = driver.GetDriverInfo()
@ -99,10 +102,13 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
l.resource = createGenericVolumeTestResource(driver, l.config, pattern)
if l.resource.volSource == nil {
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
}
}
cleanup := func() {
@ -115,6 +121,8 @@ func (t *volumeIOTestSuite) defineTests(driver TestDriver, pattern testpatterns.
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
}
It("should write files of various sizes, verify size, validate content [Slow]", func() {

View File

@ -70,6 +70,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
ns *v1.Namespace
// genericVolumeTestResource contains pv, pvc, sc, etc., owns cleaning that up
genericVolumeTestResource
intreeOps opCounts
migratedOps opCounts
}
var (
dInfo = driver.GetDriverInfo()
@ -91,6 +94,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
// Now do the more expensive test initialization.
l.config, l.testCleanup = driver.PrepareTest(f)
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
fsType := pattern.FsType
volBindMode := storagev1.VolumeBindingImmediate
@ -153,6 +157,8 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
}
// We register different tests depending on the drive

View File

@ -98,6 +98,9 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
testCleanup func()
resource *genericVolumeTestResource
intreeOps opCounts
migratedOps opCounts
}
var dInfo = driver.GetDriverInfo()
var l local
@ -119,6 +122,8 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
if l.resource.volSource == nil {
framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
}
l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName)
}
cleanup := func() {
@ -131,6 +136,8 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
l.testCleanup()
l.testCleanup = nil
}
validateMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName, l.intreeOps, l.migratedOps)
}
It("should be mountable", func() {