diff --git a/test/e2e/framework/pod/node_selection.go b/test/e2e/framework/pod/node_selection.go index ddf81dd5877..9931afa6566 100644 --- a/test/e2e/framework/pod/node_selection.go +++ b/test/e2e/framework/pod/node_selection.go @@ -48,6 +48,28 @@ func setNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSe }) } +// SetNodeAffinityTopologyRequirement sets node affinity to a specified topology +func SetNodeAffinityTopologyRequirement(nodeSelection *NodeSelection, topology map[string]string) { + if nodeSelection.Affinity == nil { + nodeSelection.Affinity = &v1.Affinity{} + } + if nodeSelection.Affinity.NodeAffinity == nil { + nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{} + } + if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{} + } + for k, v := range topology { + nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: k, Operator: v1.NodeSelectorOpIn, Values: []string{v}}, + }, + }) + + } +} + // SetAffinity sets affinity to nodeName to nodeSelection func SetAffinity(nodeSelection *NodeSelection, nodeName string) { setNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName) diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 4c967d486de..a286a8a14ca 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -370,12 +370,14 @@ func InitISCSIDriver() testsuites.TestDriver { //"ext3", "ext4", ), + TopologyKeys: []string{v1.LabelHostname}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapFsGroup: true, testsuites.CapBlock: true, testsuites.CapExec: true, testsuites.CapMultiPODs: true, + testsuites.CapTopology: true, }, }, } @@ -772,10 +774,12 @@ func InitHostPathDriver() testsuites.TestDriver { SupportedFsType: sets.NewString( "", // Default fsType ), + TopologyKeys: []string{v1.LabelHostname}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapMultiPODs: true, testsuites.CapSingleNodeVolume: true, + testsuites.CapTopology: true, }, }, } @@ -845,10 +849,12 @@ func InitHostPathSymlinkDriver() testsuites.TestDriver { SupportedFsType: sets.NewString( "", // Default fsType ), + TopologyKeys: []string{v1.LabelHostname}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapMultiPODs: true, testsuites.CapSingleNodeVolume: true, + testsuites.CapTopology: true, }, }, } @@ -1059,6 +1065,7 @@ func InitCinderDriver() testsuites.TestDriver { "", // Default fsType "ext3", ), + TopologyKeys: []string{v1.LabelZoneFailureDomain}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapFsGroup: true, @@ -1067,6 +1074,7 @@ func InitCinderDriver() testsuites.TestDriver { // Cinder supports volume limits, but the test creates large // number of volumes and times out test suites. testsuites.CapVolumeLimits: false, + testsuites.CapTopology: true, }, }, } @@ -1366,11 +1374,13 @@ func InitVSphereDriver() testsuites.TestDriver { "", // Default fsType "ext4", ), + TopologyKeys: []string{v1.LabelZoneFailureDomain}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapFsGroup: true, testsuites.CapExec: true, testsuites.CapMultiPODs: true, + testsuites.CapTopology: true, }, }, } @@ -1490,6 +1500,7 @@ func InitAzureDiskDriver() testsuites.TestDriver { "ext4", "xfs", ), + TopologyKeys: []string{v1.LabelZoneFailureDomain}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapFsGroup: true, @@ -1499,6 +1510,7 @@ func InitAzureDiskDriver() testsuites.TestDriver { // Azure supports volume limits, but the test creates large // number of volumes and times out test suites. testsuites.CapVolumeLimits: false, + testsuites.CapTopology: true, }, }, } @@ -1621,6 +1633,7 @@ func InitAwsDriver() testsuites.TestDriver { "ntfs", ), SupportedMountOption: sets.NewString("debug", "nouid32"), + TopologyKeys: []string{v1.LabelZoneFailureDomain}, Capabilities: map[testsuites.Capability]bool{ testsuites.CapPersistence: true, testsuites.CapFsGroup: true, @@ -1632,6 +1645,7 @@ func InitAwsDriver() testsuites.TestDriver { // AWS supports volume limits, but the test creates large // number of volumes and times out test suites. testsuites.CapVolumeLimits: false, + testsuites.CapTopology: true, }, }, } diff --git a/test/e2e/storage/testsuites/multivolume.go b/test/e2e/storage/testsuites/multivolume.go index 93f196dfec8..ec74f837016 100644 --- a/test/e2e/storage/testsuites/multivolume.go +++ b/test/e2e/storage/testsuites/multivolume.go @@ -180,6 +180,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter if l.config.ClientNodeName != "" { e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name) } + // For multi-node tests there must be enough nodes with the same toopology to schedule the pods + nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName} + topologyKeys := dInfo.TopologyKeys + if len(topologyKeys) != 0 { + if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil { + framework.Failf("Error setting topology requirements: %v", err) + } + } var pvcs []*v1.PersistentVolumeClaim numVols := 2 @@ -192,7 +200,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter } TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name, - e2epod.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */) + nodeSelection, pvcs, false /* sameNode */) }) // This tests below configuration (only pattern is tested): @@ -266,6 +274,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter if l.config.ClientNodeName != "" { e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name) } + // For multi-node tests there must be enough nodes with the same toopology to schedule the pods + nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName} + topologyKeys := dInfo.TopologyKeys + if len(topologyKeys) != 0 { + if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil { + framework.Failf("Error setting topology requirements: %v", err) + } + } var pvcs []*v1.PersistentVolumeClaim numVols := 2 @@ -283,7 +299,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter } TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name, - e2epod.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */) + nodeSelection, pvcs, false /* sameNode */) }) // This tests below configuration: @@ -335,6 +351,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter if l.config.ClientNodeName != "" { e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name) } + // For multi-node tests there must be enough nodes with the same toopology to schedule the pods + nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName} + topologyKeys := dInfo.TopologyKeys + if len(topologyKeys) != 0 { + if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil { + framework.Failf("Error setting topology requirements: %v", err) + } + } // Create volume testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange @@ -343,7 +367,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter // Test access to the volume from pods on different node TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name, - e2epod.NodeSelection{Name: l.config.ClientNodeName}, resource.Pvc, numPods, false /* sameNode */) + nodeSelection, resource.Pvc, numPods, false /* sameNode */) }) } @@ -504,3 +528,56 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed) } } + +// getCurrentTopologies() goes through all Nodes and returns unique driver topologies and count of Nodes per topology +func getCurrentTopologiesNumber(cs clientset.Interface, nodes *v1.NodeList, keys []string) ([]topology, []int, error) { + topos := []topology{} + topoCount := []int{} + + // TODO: scale? + for _, n := range nodes.Items { + topo := map[string]string{} + for _, k := range keys { + v, ok := n.Labels[k] + if ok { + topo[k] = v + } + } + + found := false + for i, existingTopo := range topos { + if topologyEqual(existingTopo, topo) { + found = true + topoCount[i]++ + break + } + } + if !found { + framework.Logf("found topology %v", topo) + topos = append(topos, topo) + topoCount = append(topoCount, 1) + } + } + return topos, topoCount, nil +} + +// ensureTopologyRequirements sets nodeSelection affinity according to given topology keys for drivers that provide them +func ensureTopologyRequirements(nodeSelection *e2epod.NodeSelection, nodes *v1.NodeList, cs clientset.Interface, topologyKeys []string, minCount int) error { + topologyList, topologyCount, err := getCurrentTopologiesNumber(cs, nodes, topologyKeys) + if err != nil { + return err + } + suitableTopologies := []topology{} + for i, topo := range topologyList { + if topologyCount[i] >= minCount { + suitableTopologies = append(suitableTopologies, topo) + } + } + if len(suitableTopologies) == 0 { + framework.Skipf("No topology with at least %d nodes found - skipping", minCount) + } + // Take the first suitable topology + e2epod.SetNodeAffinityTopologyRequirement(nodeSelection, suitableTopologies[0]) + + return nil +}