From 10f652abec7f14a3b26ccf523ead2968e8fb3a7f Mon Sep 17 00:00:00 2001 From: Deep Debroy Date: Tue, 28 Aug 2018 12:35:43 -0700 Subject: [PATCH] E2E tests for allowedTopologies and waitForConsumer for GCE PD and RePD Signed-off-by: Deep Debroy --- test/e2e/storage/regional_pd.go | 114 ++++++++++++- test/e2e/storage/volume_provisioning.go | 218 +++++++++++++++--------- 2 files changed, 249 insertions(+), 83 deletions(-) diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go index c5556e7340d..6c9e0f9cddc 100644 --- a/test/e2e/storage/regional_pd.go +++ b/test/e2e/storage/regional_pd.go @@ -34,6 +34,7 @@ import ( clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/kubelet/apis" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" @@ -65,10 +66,21 @@ var _ = utils.SIGDescribe("Regional PD", func() { testVolumeProvisioning(c, ns) }) + It("should provision storage with delayed binding [Slow] [Feature:DynamicProvisioningScheduling]", func() { + testRegionalDelayedBinding(c, ns) + }) + + It("should provision storage in the allowedTopologies [Slow] [Feature:DynamicProvisioningScheduling]", func() { + testRegionalAllowedTopologies(c, ns) + }) + + It("should provision storage in the allowedTopologies with delayed binding [Slow] [Feature:DynamicProvisioningScheduling]", func() { + testRegionalAllowedTopologiesWithDelayedBinding(c, ns) + }) + It("should failover to a different zone when all nodes in one zone become unreachable [Slow] [Disruptive]", func() { testZonalFailover(c, ns) }) - }) }) @@ -264,6 +276,94 @@ func testZonalFailover(c clientset.Interface, ns string) { } +func testRegionalDelayedBinding(c clientset.Interface, ns string) { + test := storageClassTest{ + name: "Regional PD storage class with waitForFirstConsumer test on GCE", + provisioner: "kubernetes.io/gce-pd", + parameters: map[string]string{ + "type": "pd-standard", + "replication-type": "regional-pd", + }, + claimSize: "2Gi", + delayBinding: true, + } + + suffix := "delayed-regional" + class := newStorageClass(test, ns, suffix) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv, node := testBindingWaitForFirstConsumer(c, claim, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + checkZoneFromLabelAndAffinity(pv, zone, false) +} + +func testRegionalAllowedTopologies(c clientset.Interface, ns string) { + test := storageClassTest{ + name: "Regional PD storage class with allowedTopologies test on GCE", + provisioner: "kubernetes.io/gce-pd", + parameters: map[string]string{ + "type": "pd-standard", + "replication-type": "regional-pd", + }, + claimSize: "2Gi", + expectedSize: "2Gi", + } + + suffix := "topo-regional" + class := newStorageClass(test, ns, suffix) + zones := getTwoRandomZones(c) + addAllowedTopologiesToStorageClass(c, class, zones) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv := testDynamicProvisioning(test, c, claim, class) + checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true) +} + +func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) { + test := storageClassTest{ + name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE", + provisioner: "kubernetes.io/gce-pd", + parameters: map[string]string{ + "type": "pd-standard", + "replication-type": "regional-pd", + }, + claimSize: "2Gi", + delayBinding: true, + } + + suffix := "topo-delayed-regional" + class := newStorageClass(test, ns, suffix) + topoZones := getTwoRandomZones(c) + addAllowedTopologiesToStorageClass(c, class, topoZones) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv, node := testBindingWaitForFirstConsumer(c, claim, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + zoneFound := false + for _, zone := range topoZones { + if zone == nodeZone { + zoneFound = true + break + } + } + if !zoneFound { + framework.Failf("zones specified in AllowedTopologies: %v does not contain zone of node where PV got provisioned: %s", topoZones, nodeZone) + } + checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true) +} + func getPVC(c clientset.Interface, ns string, pvcLabels map[string]string) *v1.PersistentVolumeClaim { selector := labels.Set(pvcLabels).AsSelector() options := metav1.ListOptions{LabelSelector: selector.String()} @@ -284,6 +384,18 @@ func getPod(c clientset.Interface, ns string, podLabels map[string]string) *v1.P return &podList.Items[0] } +func addAllowedTopologiesToStorageClass(c clientset.Interface, sc *storage.StorageClass, zones []string) { + term := v1.TopologySelectorTerm{ + MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{ + { + Key: kubeletapis.LabelZoneFailureDomain, + Values: zones, + }, + }, + } + sc.AllowedTopologies = append(sc.AllowedTopologies, term) +} + // Generates the spec of a StatefulSet with 1 replica that mounts a Regional PD. func newStatefulSet(claimTemplate *v1.PersistentVolumeClaim, ns string) (sts *appsv1.StatefulSet, svc *v1.Service, labels map[string]string) { var replicas int32 = 1 diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index 229e1954e80..8853b725431 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -44,6 +44,7 @@ import ( clientset "k8s.io/client-go/kubernetes" storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" + volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -223,9 +224,14 @@ func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.Persi return pv, node } +func checkZoneFromLabelAndAffinity(pv *v1.PersistentVolume, zone string, matchZone bool) { + checkZonesFromLabelAndAffinity(pv, sets.NewString(zone), matchZone) +} + // checkZoneLabelAndAffinity checks the LabelZoneFailureDomain label of PV and terms -// with key LabelZoneFailureDomain in PV's node affinity match zone -func checkZoneLabelAndAffinity(pv *v1.PersistentVolume, zone string) { +// with key LabelZoneFailureDomain in PV's node affinity contains zone +// matchZones is used to indicate if zones should match perfectly +func checkZonesFromLabelAndAffinity(pv *v1.PersistentVolume, zones sets.String, matchZones bool) { By("checking PV's zone label and node affinity terms match expected zone") if pv == nil { framework.Failf("nil pv passed") @@ -235,14 +241,19 @@ func checkZoneLabelAndAffinity(pv *v1.PersistentVolume, zone string) { framework.Failf("label %s not found on PV", kubeletapis.LabelZoneFailureDomain) } - if zone != pvLabel { - framework.Failf("value of %s label for PV: %s does not match expected zone: %s", kubeletapis.LabelZoneFailureDomain, pvLabel, zone) + zonesFromLabel, err := volumeutil.LabelZonesToSet(pvLabel) + if err != nil { + framework.Failf("unable to parse zone labels %s: %v", pvLabel, err) + } + if matchZones && !zonesFromLabel.Equal(zones) { + framework.Failf("value[s] of %s label for PV: %v does not match expected zone[s]: %v", kubeletapis.LabelZoneFailureDomain, zonesFromLabel, zones) + } + if !matchZones && !zonesFromLabel.IsSuperset(zones) { + framework.Failf("value[s] of %s label for PV: %v does not contain expected zone[s]: %v", kubeletapis.LabelZoneFailureDomain, zonesFromLabel, zones) } - if pv.Spec.NodeAffinity == nil { framework.Failf("node affinity not found in PV spec %v", pv.Spec) } - if len(pv.Spec.NodeAffinity.Required.NodeSelectorTerms) == 0 { framework.Failf("node selector terms not found in PV spec %v", pv.Spec) } @@ -250,17 +261,18 @@ func checkZoneLabelAndAffinity(pv *v1.PersistentVolume, zone string) { for _, term := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms { keyFound := false for _, r := range term.MatchExpressions { - if r.Key == kubeletapis.LabelZoneFailureDomain { - keyFound = true - for _, val := range r.Values { - if zone == val { - framework.Logf("expected zone %s detected", val) - } else { - framework.Failf("zone %s does not match expected zone %s", val, zone) - } - } - break + if r.Key != kubeletapis.LabelZoneFailureDomain { + continue } + keyFound = true + zonesFromNodeAffinity := sets.NewString(r.Values...) + if matchZones && !zonesFromNodeAffinity.Equal(zones) { + framework.Failf("zones from NodeAffinity of PV: %v does not equal expected zone[s]: %v", zonesFromNodeAffinity, zones) + } + if !matchZones && !zonesFromNodeAffinity.IsSuperset(zones) { + framework.Failf("zones from NodeAffinity of PV: %v does not contain expected zone[s]: %v", zonesFromNodeAffinity, zones) + } + break } if !keyFound { framework.Failf("label %s not found in term %v", kubeletapis.LabelZoneFailureDomain, term) @@ -973,81 +985,123 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { }) }) Describe("DynamicProvisioner delayed binding [Feature:DynamicProvisioningScheduling] [Slow]", func() { - It("should create persistent volume in the same zone as node after a pod mounting the claim is started", func() { - framework.SkipUnlessProviderIs("aws") - - By("creating a claim with class with waitForFirstConsumer") - test := storageClassTest{ - name: "Delayed binding EBS storage class test", - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - delayBinding: true, + It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() { + tests := []storageClassTest{ + { + name: "Delayed binding EBS storage class test", + cloudProviders: []string{"aws"}, + provisioner: "kubernetes.io/aws-ebs", + claimSize: "2Gi", + delayBinding: true, + }, + { + name: "Delayed binding GCE PD storage class test", + cloudProviders: []string{"gce", "gke"}, + provisioner: "kubernetes.io/gce-pd", + claimSize: "2Gi", + delayBinding: true, + }, } - suffix := "delayed-ebs" - class := newStorageClass(test, ns, suffix) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") + for _, test := range tests { + if !framework.ProviderIs(test.cloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + continue + } + By("creating a claim with class with waitForFirstConsumer") + suffix := "delayed" + class := newStorageClass(test, ns, suffix) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv, node := testBindingWaitForFirstConsumer(c, claim, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + checkZoneFromLabelAndAffinity(pv, zone, true) } - zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - checkZoneLabelAndAffinity(pv, zone) }) }) - Describe("DynamicProvisioner allowedTopology [Feature:DynamicProvisioningScheduling]", func() { - It("should create persistent volume in the zone specified in allowedTopology of storageclass", func() { - framework.SkipUnlessProviderIs("aws") - - By("creating a claim with class with allowedTopology set") - test := storageClassTest{ - name: "Delayed binding EBS storage class test", - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - expectedSize: "2Gi", + Describe("DynamicProvisioner allowedTopologies [Feature:DynamicProvisioningScheduling]", func() { + It("should create persistent volume in the zone specified in allowedTopologies of storageclass", func() { + tests := []storageClassTest{ + { + name: "AllowedTopologies EBS storage class test", + cloudProviders: []string{"aws"}, + provisioner: "kubernetes.io/aws-ebs", + claimSize: "2Gi", + expectedSize: "2Gi", + }, + { + name: "AllowedTopologies GCE PD storage class test", + cloudProviders: []string{"gce", "gke"}, + provisioner: "kubernetes.io/gce-pd", + claimSize: "2Gi", + expectedSize: "2Gi", + }, + } + for _, test := range tests { + if !framework.ProviderIs(test.cloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + continue + } + By("creating a claim with class with allowedTopologies set") + suffix := "topology" + class := newStorageClass(test, ns, suffix) + zone := getRandomCloudZone(c) + addSingleZoneAllowedTopologyToStorageClass(c, class, zone) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv := testDynamicProvisioning(test, c, claim, class) + checkZoneFromLabelAndAffinity(pv, zone, true) } - suffix := "topo-ebs" - class := newStorageClass(test, ns, suffix) - zone := getRandomCloudZone(c) - addSingleZoneAllowedTopologyToStorageClass(c, class, zone) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv := testDynamicProvisioning(test, c, claim, class) - checkZoneLabelAndAffinity(pv, zone) }) }) - Describe("DynamicProvisioner delayed binding with allowedTopology [Feature:DynamicProvisioningScheduling] [Slow]", func() { - It("should create persistent volume in the same zone as specified in allowedTopology after a pod mounting the claim is started", func() { - framework.SkipUnlessProviderIs("aws") - - By("creating a claim with class with waitForFirstConsumer") - test := storageClassTest{ - name: "Delayed binding EBS storage class test", - provisioner: "kubernetes.io/aws-ebs", - claimSize: "2Gi", - delayBinding: true, + Describe("DynamicProvisioner delayed binding with allowedTopologies [Feature:DynamicProvisioningScheduling] [Slow]", func() { + It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() { + tests := []storageClassTest{ + { + name: "AllowedTopologies and delayed binding EBS storage class test", + cloudProviders: []string{"aws"}, + provisioner: "kubernetes.io/aws-ebs", + claimSize: "2Gi", + delayBinding: true, + }, + { + name: "AllowedTopologies and delayed binding GCE PD storage class test", + cloudProviders: []string{"gce", "gke"}, + provisioner: "kubernetes.io/gce-pd", + claimSize: "2Gi", + delayBinding: true, + }, } - suffix := "delayed-topo-ebs" - class := newStorageClass(test, ns, suffix) - topoZone := getRandomCloudZone(c) - addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) - claim := newClaim(test, ns, suffix) - claim.Spec.StorageClassName = &class.Name - pv, node := testBindingWaitForFirstConsumer(c, claim, class) - if node == nil { - framework.Failf("unexpected nil node found") + for _, test := range tests { + if !framework.ProviderIs(test.cloudProviders...) { + framework.Logf("Skipping %q: cloud providers is not %v", test.name, test.cloudProviders) + continue + } + By("creating a claim with class with WaitForFirstConsumer and allowedTopologies") + suffix := "delayed-topo" + class := newStorageClass(test, ns, suffix) + topoZone := getRandomCloudZone(c) + addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone) + claim := newClaim(test, ns, suffix) + claim.Spec.StorageClassName = &class.Name + pv, node := testBindingWaitForFirstConsumer(c, claim, class) + if node == nil { + framework.Failf("unexpected nil node found") + } + nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] + if !ok { + framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) + } + if topoZone != nodeZone { + framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone) + } + checkZoneFromLabelAndAffinity(pv, topoZone, true) } - nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain] - if !ok { - framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain) - } - if topoZone != nodeZone { - framework.Failf("zone specified in AllowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone) - } - checkZoneLabelAndAffinity(pv, topoZone) }) })