Merge pull request #32485 from jsafrane/e2e-provisioning-class

Automatic merge from submit-queue

Add e2e tests for storageclass

- test pd-ssd and pd-standard on GCE,
- test all four volume types and encryption on AWS
- test just the default volume type on OpenStack (right now, there is no API
  to get list of them)

These tests are quite slow, e.g. there are two tests on AWS that has to run mkfs.ext4 on 500 GB magnetic drive with low IOPS, which takes ~3-4 minutes each.

pkg/cloudprovider/providers/gce/gce.go

@@ -2562,6 +2562,21 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
 	return labels, nil
 }
 
+// TestDisk checks that a disk has given type. It should be used only for
+// testing!
+func (gce *GCECloud) TestDisk(diskName, volumeType string) error {
+	disk, err := gce.getDiskByNameUnknownZone(diskName)
+	if err != nil {
+		return err
+	}
+
+	if strings.HasSuffix(disk.Type, volumeType) {
+		return nil
+	}
+
+	return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
+}
+
 func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error {
 	instanceName := mapNodeNameToInstanceName(nodeName)
 	instance, err := gce.getInstanceByName(instanceName)
@@ -2645,6 +2660,7 @@ func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, err
 			Zone: lastComponent(disk.Zone),
 			Name: disk.Name,
 			Kind: disk.Kind,
+			Type: disk.Type,
 		}
 		return d, nil
 	}
@@ -2765,6 +2781,7 @@ type gceDisk struct {
 	Zone string
 	Name string
 	Kind string
+	Type string
 }
 
 // Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found

test/e2e/volume_provisioning.go

@@ -17,8 +17,13 @@ limitations under the License.
 package e2e
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/api/unversioned"
@@ -30,16 +35,35 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const (
+type storageClassTest struct {
-	// Requested size of the volume
+	name           string
-	requestedSize = "1500Mi"
+	cloudProviders []string
-	// Expected size of the volume is 2GiB, because all three supported cloud
+	provisioner    string
-	// providers allocate volumes in 1GiB chunks.
+	parameters     map[string]string
-	expectedSize = "2Gi"
+	claimSize      string
-)
+	expectedSize   string
+	pvCheck        func(volume *api.PersistentVolume) error
+}
 
-func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeClaim) {
+func testDynamicProvisioning(t storageClassTest, client *client.Client, claim *api.PersistentVolumeClaim, class *storage.StorageClass) {
-	err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+	if class != nil {
+		By("creating a StorageClass " + class.Name)
+		class, err := client.Storage().StorageClasses().Create(class)
+		defer func() {
+			framework.Logf("deleting storage class %s", class.Name)
+			client.Storage().StorageClasses().Delete(class.Name)
+		}()
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("creating a claim")
+	claim, err := client.PersistentVolumeClaims(claim.Namespace).Create(claim)
+	defer func() {
+		framework.Logf("deleting claim %s/%s", claim.Namespace, claim.Name)
+		client.PersistentVolumeClaims(claim.Namespace).Delete(claim.Name)
+	}()
+	Expect(err).NotTo(HaveOccurred())
+	err = framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("checking the claim")
@@ -52,21 +76,28 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
 	Expect(err).NotTo(HaveOccurred())
 
 	// Check sizes
-	expectedCapacity := resource.MustParse(expectedSize)
+	expectedCapacity := resource.MustParse(t.expectedSize)
 	pvCapacity := pv.Spec.Capacity[api.ResourceName(api.ResourceStorage)]
 	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()))
 
-	requestedCapacity := resource.MustParse(requestedSize)
+	requestedCapacity := resource.MustParse(t.claimSize)
 	claimCapacity := claim.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)]
 	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()))
 
 	// Check PV properties
+	By("checking the PV")
 	Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(api.PersistentVolumeReclaimDelete))
 	expectedAccessModes := []api.PersistentVolumeAccessMode{api.ReadWriteOnce}
 	Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
 	Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
 	Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
 
+	// Run the checker
+	if t.pvCheck != nil {
+		err = t.pvCheck(pv)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
 	// We start two pods:
 	// - The first writes 'hello word' to the /mnt/test (= the volume).
 	// - The second one runs grep 'hello world' on /mnt/test.
@@ -101,6 +132,54 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
 	framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
 }
 
+// checkAWSEBS checks properties of an AWS EBS. Test framework does not
+// instantiate full AWS provider, therefore we need use ec2 API directly.
+func checkAWSEBS(volume *api.PersistentVolume, volumeType string, encrypted bool) error {
+	diskName := volume.Spec.AWSElasticBlockStore.VolumeID
+
+	client := ec2.New(session.New())
+	tokens := strings.Split(diskName, "/")
+	volumeID := tokens[len(tokens)-1]
+
+	request := &ec2.DescribeVolumesInput{
+		VolumeIds: []*string{&volumeID},
+	}
+	info, err := client.DescribeVolumes(request)
+	if err != nil {
+		return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
+	}
+	if len(info.Volumes) == 0 {
+		return fmt.Errorf("no volumes found for volume %q", volumeID)
+	}
+	if len(info.Volumes) > 1 {
+		return fmt.Errorf("multiple volumes found for volume %q", volumeID)
+	}
+
+	awsVolume := info.Volumes[0]
+	if awsVolume.VolumeType == nil {
+		return fmt.Errorf("expected volume type %q, got nil", volumeType)
+	}
+	if *awsVolume.VolumeType != volumeType {
+		return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
+	}
+	if encrypted && awsVolume.Encrypted == nil {
+		return fmt.Errorf("expected encrypted volume, got no encryption")
+	}
+	if encrypted && !*awsVolume.Encrypted {
+		return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
+	}
+	return nil
+}
+
+func checkGCEPD(volume *api.PersistentVolume, volumeType string) error {
+	cloud, err := getGCECloud()
+	if err != nil {
+		return err
+	}
+	diskName := volume.Spec.GCEPersistentDisk.PDName
+	return cloud.TestDisk(diskName, volumeType)
+}
+
 var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 	f := framework.NewDefaultFramework("volume-provisioning")
 
@@ -114,45 +193,170 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 	})
 
 	framework.KubeDescribe("DynamicProvisioner", func() {
-		It("should create and delete persistent volumes [Slow]", func() {
+		// This test checks that dynamic provisioning can provision a volume
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
+		// that can be used to persist data among pods.
 
-			By("creating a StorageClass")
+		tests := []storageClassTest{
-			class := newStorageClass()
+			{
-			_, err := c.Storage().StorageClasses().Create(class)
+				"should provision SSD PD on GCE/GKE",
-			defer c.Storage().StorageClasses().Delete(class.Name)
+				[]string{"gce", "gke"},
-			Expect(err).NotTo(HaveOccurred())
+				"kubernetes.io/gce-pd",
+				map[string]string{
+					"type": "pd-ssd",
+					// Check that GCE can parse "zone" parameter, however
+					// we can't create PDs in different than default zone
+					// as we don't know if we're running with Multizone=true
+					"zone": framework.TestContext.CloudConfig.Zone,
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkGCEPD(volume, "pd-ssd")
+				},
+			},
+			{
+				"should provision HDD PD on GCE/GKE",
+				[]string{"gce", "gke"},
+				"kubernetes.io/gce-pd",
+				map[string]string{
+					"type": "pd-standard",
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkGCEPD(volume, "pd-standard")
+				},
+			},
+			// AWS
+			{
+				"should provision gp2 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "gp2",
+					// Check that AWS can parse "zone" parameter, however
+					// we can't create PDs in different than default zone
+					// as we don't know zone names
+					"zone": framework.TestContext.CloudConfig.Zone,
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "gp2", false)
+				},
+			},
+			{
+				"should provision io1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type":      "io1",
+					"iopsPerGB": "50",
+				},
+				"3.5Gi",
+				"4Gi", // 4 GiB is minimum for io1
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "io1", false)
+				},
+			},
+			{
+				"should provision sc1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "sc1",
+				},
+				"500Gi", // minimum for sc1
+				"500Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "sc1", false)
+				},
+			},
+			{
+				"should provision st1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "st1",
+				},
+				"500Gi", // minimum for st1
+				"500Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "st1", false)
+				},
+			},
+			{
+				"should provision encrypted EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"encrypted": "true",
+				},
+				"1Gi",
+				"1Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "gp2", true)
+				},
+			},
+			// OpenStack generic tests (works on all OpenStack deployments)
+			{
+				"should provision generic Cinder volume on OpenStack",
+				[]string{"openstack"},
+				"kubernetes.io/cinder",
+				map[string]string{},
+				"1.5Gi",
+				"2Gi",
+				nil, // there is currently nothing to check on OpenStack
+			},
+			{
+				"should provision Cinder volume with empty volume type and zone on OpenStack",
+				[]string{"openstack"},
+				"kubernetes.io/cinder",
+				map[string]string{
+					"type":         "",
+					"availability": "",
+				},
+				"1.5Gi",
+				"2Gi",
+				nil, // there is currently nothing to check on OpenStack
+			},
+		}
 
-			By("creating a claim with a dynamic provisioning annotation")
+		for i, t := range tests {
-			claim := newClaim(ns, false)
+			// Beware of clojure, use local variables instead of those from
-			defer func() {
+			// outer scope
-				c.PersistentVolumeClaims(ns).Delete(claim.Name)
+			test := t
-			}()
+			suffix := fmt.Sprintf("%d", i)
-			claim, err = c.PersistentVolumeClaims(ns).Create(claim)
+			It(test.name, func() {
-			Expect(err).NotTo(HaveOccurred())
+				if len(t.cloudProviders) > 0 {
+					framework.SkipUnlessProviderIs(test.cloudProviders...)
+				}
 
-			testDynamicProvisioning(c, claim)
+				class := newStorageClass(test, suffix)
+				claim := newClaim(test, ns, suffix, false)
+				testDynamicProvisioning(test, c, claim, class)
 			})
+		}
 	})
 
 	framework.KubeDescribe("DynamicProvisioner Alpha", func() {
-		It("should create and delete alpha persistent volumes [Slow]", func() {
+		It("should provision alpha volumes [Slow]", func() {
 			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
 
 			By("creating a claim with an alpha dynamic provisioning annotation")
-			claim := newClaim(ns, true)
+			test := storageClassTest{
-			defer func() {
+				name:         "alpha test",
-				c.PersistentVolumeClaims(ns).Delete(claim.Name)
+				claimSize:    "1500Mi",
-			}()
+				expectedSize: "2Gi",
-			claim, err := c.PersistentVolumeClaims(ns).Create(claim)
+			}
-			Expect(err).NotTo(HaveOccurred())
 
-			testDynamicProvisioning(c, claim)
+			claim := newClaim(test, ns, "", true)
+			testDynamicProvisioning(test, c, claim, nil)
 		})
 	})
 })
 
-func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
+func newClaim(t storageClassTest, ns, suffix string, alpha bool) *api.PersistentVolumeClaim {
 	claim := api.PersistentVolumeClaim{
 		ObjectMeta: api.ObjectMeta{
 			GenerateName: "pvc-",
@@ -164,7 +368,7 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
 			},
 			Resources: api.ResourceRequirements{
 				Requests: api.ResourceList{
-					api.ResourceName(api.ResourceStorage): resource.MustParse(requestedSize),
+					api.ResourceName(api.ResourceStorage): resource.MustParse(t.claimSize),
 				},
 			},
 		},
@@ -176,9 +380,8 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
 		}
 	} else {
 		claim.Annotations = map[string]string{
-			"volume.beta.kubernetes.io/storage-class": "fast",
+			"volume.beta.kubernetes.io/storage-class": "myclass-" + suffix,
 		}
-
 	}
 
 	return &claim
@@ -224,32 +427,22 @@ func runInPodWithVolume(c *client.Client, ns, claimName, command string) {
 		},
 	}
 	pod, err := c.Pods(ns).Create(pod)
+	framework.ExpectNoError(err, "Failed to create pod: %v", err)
 	defer func() {
 		framework.ExpectNoError(c.Pods(ns).Delete(pod.Name, nil))
 	}()
-	framework.ExpectNoError(err, "Failed to create pod: %v", err)
 	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
 }
 
-func newStorageClass() *storage.StorageClass {
+func newStorageClass(t storageClassTest, suffix string) *storage.StorageClass {
-	var pluginName string
-
-	switch {
-	case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
-		pluginName = "kubernetes.io/gce-pd"
-	case framework.ProviderIs("aws"):
-		pluginName = "kubernetes.io/aws-ebs"
-	case framework.ProviderIs("openstack"):
-		pluginName = "kubernetes.io/cinder"
-	}
-
 	return &storage.StorageClass{
 		TypeMeta: unversioned.TypeMeta{
 			Kind: "StorageClass",
 		},
 		ObjectMeta: api.ObjectMeta{
-			Name: "fast",
+			Name: "myclass-" + suffix,
 		},
-		Provisioner: pluginName,
+		Provisioner: t.provisioner,
+		Parameters:  t.parameters,
 	}
 }