Add e2e tests for storageclass

- test pd-ssd and pd-standard on GCE,
- test all four volume types on AWS
- test just the default volume type on OpenStack (right now, there is no API
  to get list of them)

pkg/cloudprovider/providers/gce/gce.go

@@ -2562,6 +2562,21 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
 	return labels, nil
 }
 
+// TestDisk checks that a disk has given type. It should be used only for
+// testing!
+func (gce *GCECloud) TestDisk(diskName, volumeType string) error {
+	disk, err := gce.getDiskByNameUnknownZone(diskName)
+	if err != nil {
+		return err
+	}
+
+	if strings.HasSuffix(disk.Type, volumeType) {
+		return nil
+	}
+
+	return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
+}
+
 func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error {
 	instanceName := mapNodeNameToInstanceName(nodeName)
 	instance, err := gce.getInstanceByName(instanceName)
@@ -2645,6 +2660,7 @@ func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, err
 			Zone: lastComponent(disk.Zone),
 			Name: disk.Name,
 			Kind: disk.Kind,
+			Type: disk.Type,
 		}
 		return d, nil
 	}
@@ -2765,6 +2781,7 @@ type gceDisk struct {
 	Zone string
 	Name string
 	Kind string
+	Type string
 }
 
 // Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found

test/e2e/volume_provisioning.go

@@ -17,8 +17,13 @@ limitations under the License.
 package e2e
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/api/unversioned"
@@ -30,16 +35,35 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const (
+type storageClassTest struct {
-	// Requested size of the volume
+	name           string
-	requestedSize = "1500Mi"
+	cloudProviders []string
-	// Expected size of the volume is 2GiB, because all three supported cloud
+	provisioner    string
-	// providers allocate volumes in 1GiB chunks.
+	parameters     map[string]string
-	expectedSize = "2Gi"
+	claimSize      string
-)
+	expectedSize   string
+	pvCheck        func(volume *api.PersistentVolume) error
+}
 
-func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeClaim) {
+func testDynamicProvisioning(t storageClassTest, client *client.Client, claim *api.PersistentVolumeClaim, class *storage.StorageClass) {
-	err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+	if class != nil {
+		By("creating a StorageClass " + class.Name)
+		class, err := client.Storage().StorageClasses().Create(class)
+		defer func() {
+			framework.Logf("deleting storage class %s", class.Name)
+			client.Storage().StorageClasses().Delete(class.Name)
+		}()
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("creating a claim")
+	claim, err := client.PersistentVolumeClaims(claim.Namespace).Create(claim)
+	defer func() {
+		framework.Logf("deleting claim %s/%s", claim.Namespace, claim.Name)
+		client.PersistentVolumeClaims(claim.Namespace).Delete(claim.Name)
+	}()
+	Expect(err).NotTo(HaveOccurred())
+	err = framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("checking the claim")
@@ -52,21 +76,28 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
 	Expect(err).NotTo(HaveOccurred())
 
 	// Check sizes
-	expectedCapacity := resource.MustParse(expectedSize)
+	expectedCapacity := resource.MustParse(t.expectedSize)
 	pvCapacity := pv.Spec.Capacity[api.ResourceName(api.ResourceStorage)]
 	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()))
 
-	requestedCapacity := resource.MustParse(requestedSize)
+	requestedCapacity := resource.MustParse(t.claimSize)
 	claimCapacity := claim.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)]
 	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()))
 
 	// Check PV properties
+	By("checking the PV")
 	Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(api.PersistentVolumeReclaimDelete))
 	expectedAccessModes := []api.PersistentVolumeAccessMode{api.ReadWriteOnce}
 	Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
 	Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
 	Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
 
+	// Run the checker
+	if t.pvCheck != nil {
+		err = t.pvCheck(pv)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
 	// We start two pods:
 	// - The first writes 'hello word' to the /mnt/test (= the volume).
 	// - The second one runs grep 'hello world' on /mnt/test.
@@ -101,6 +132,54 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
 	framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
 }
 
+// checkAWSEBS checks properties of an AWS EBS. Test framework does not
+// instantiate full AWS provider, therefore we need use ec2 API directly.
+func checkAWSEBS(volume *api.PersistentVolume, volumeType string, encrypted bool) error {
+	diskName := volume.Spec.AWSElasticBlockStore.VolumeID
+
+	client := ec2.New(session.New())
+	tokens := strings.Split(diskName, "/")
+	volumeID := tokens[len(tokens)-1]
+
+	request := &ec2.DescribeVolumesInput{
+		VolumeIds: []*string{&volumeID},
+	}
+	info, err := client.DescribeVolumes(request)
+	if err != nil {
+		return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
+	}
+	if len(info.Volumes) == 0 {
+		return fmt.Errorf("no volumes found for volume %q", volumeID)
+	}
+	if len(info.Volumes) > 1 {
+		return fmt.Errorf("multiple volumes found for volume %q", volumeID)
+	}
+
+	awsVolume := info.Volumes[0]
+	if awsVolume.VolumeType == nil {
+		return fmt.Errorf("expected volume type %q, got nil", volumeType)
+	}
+	if *awsVolume.VolumeType != volumeType {
+		return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
+	}
+	if encrypted && awsVolume.Encrypted == nil {
+		return fmt.Errorf("expected encrypted volume, got no encryption")
+	}
+	if encrypted && !*awsVolume.Encrypted {
+		return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
+	}
+	return nil
+}
+
+func checkGCEPD(volume *api.PersistentVolume, volumeType string) error {
+	cloud, err := getGCECloud()
+	if err != nil {
+		return err
+	}
+	diskName := volume.Spec.GCEPersistentDisk.PDName
+	return cloud.TestDisk(diskName, volumeType)
+}
+
 var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 	f := framework.NewDefaultFramework("volume-provisioning")
 
@@ -114,45 +193,170 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 	})
 
 	framework.KubeDescribe("DynamicProvisioner", func() {
-		It("should create and delete persistent volumes [Slow]", func() {
+		// This test checks that dynamic provisioning can provision a volume
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
+		// that can be used to persist data among pods.
 
-			By("creating a StorageClass")
+		tests := []storageClassTest{
-			class := newStorageClass()
+			{
-			_, err := c.Storage().StorageClasses().Create(class)
+				"should provision SSD PD on GCE/GKE",
-			defer c.Storage().StorageClasses().Delete(class.Name)
+				[]string{"gce", "gke"},
-			Expect(err).NotTo(HaveOccurred())
+				"kubernetes.io/gce-pd",
+				map[string]string{
+					"type": "pd-ssd",
+					// Check that GCE can parse "zone" parameter, however
+					// we can't create PDs in different than default zone
+					// as we don't know if we're running with Multizone=true
+					"zone": framework.TestContext.CloudConfig.Zone,
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkGCEPD(volume, "pd-ssd")
+				},
+			},
+			{
+				"should provision HDD PD on GCE/GKE",
+				[]string{"gce", "gke"},
+				"kubernetes.io/gce-pd",
+				map[string]string{
+					"type": "pd-standard",
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkGCEPD(volume, "pd-standard")
+				},
+			},
+			// AWS
+			{
+				"should provision gp2 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "gp2",
+					// Check that AWS can parse "zone" parameter, however
+					// we can't create PDs in different than default zone
+					// as we don't know zone names
+					"zone": framework.TestContext.CloudConfig.Zone,
+				},
+				"1.5Gi",
+				"2Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "gp2", false)
+				},
+			},
+			{
+				"should provision io1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type":      "io1",
+					"iopsPerGB": "50",
+				},
+				"3.5Gi",
+				"4Gi", // 4 GiB is minimum for io1
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "io1", false)
+				},
+			},
+			{
+				"should provision sc1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "sc1",
+				},
+				"500Gi", // minimum for sc1
+				"500Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "sc1", false)
+				},
+			},
+			{
+				"should provision st1 EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"type": "st1",
+				},
+				"500Gi", // minimum for st1
+				"500Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "st1", false)
+				},
+			},
+			{
+				"should provision encrypted EBS on AWS",
+				[]string{"aws"},
+				"kubernetes.io/aws-ebs",
+				map[string]string{
+					"encrypted": "true",
+				},
+				"1Gi",
+				"1Gi",
+				func(volume *api.PersistentVolume) error {
+					return checkAWSEBS(volume, "gp2", true)
+				},
+			},
+			// OpenStack generic tests (works on all OpenStack deployments)
+			{
+				"should provision generic Cinder volume on OpenStack",
+				[]string{"openstack"},
+				"kubernetes.io/cinder",
+				map[string]string{},
+				"1.5Gi",
+				"2Gi",
+				nil, // there is currently nothing to check on OpenStack
+			},
+			{
+				"should provision Cinder volume with empty volume type and zone on OpenStack",
+				[]string{"openstack"},
+				"kubernetes.io/cinder",
+				map[string]string{
+					"type":         "",
+					"availability": "",
+				},
+				"1.5Gi",
+				"2Gi",
+				nil, // there is currently nothing to check on OpenStack
+			},
+		}
 
-			By("creating a claim with a dynamic provisioning annotation")
+		for i, t := range tests {
-			claim := newClaim(ns, false)
+			// Beware of clojure, use local variables instead of those from
-			defer func() {
+			// outer scope
-				c.PersistentVolumeClaims(ns).Delete(claim.Name)
+			test := t
-			}()
+			suffix := fmt.Sprintf("%d", i)
-			claim, err = c.PersistentVolumeClaims(ns).Create(claim)
+			It(test.name, func() {
-			Expect(err).NotTo(HaveOccurred())
+				if len(t.cloudProviders) > 0 {
+					framework.SkipUnlessProviderIs(test.cloudProviders...)
+				}
 
-			testDynamicProvisioning(c, claim)
+				class := newStorageClass(test, suffix)
-		})
+				claim := newClaim(test, ns, suffix, false)
+				testDynamicProvisioning(test, c, claim, class)
+			})
+		}
 	})
 
 	framework.KubeDescribe("DynamicProvisioner Alpha", func() {
-		It("should create and delete alpha persistent volumes [Slow]", func() {
+		It("should provision alpha volumes [Slow]", func() {
 			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
 
 			By("creating a claim with an alpha dynamic provisioning annotation")
-			claim := newClaim(ns, true)
+			test := storageClassTest{
-			defer func() {
+				name:         "alpha test",
-				c.PersistentVolumeClaims(ns).Delete(claim.Name)
+				claimSize:    "1500Mi",
-			}()
+				expectedSize: "2Gi",
-			claim, err := c.PersistentVolumeClaims(ns).Create(claim)
+			}
-			Expect(err).NotTo(HaveOccurred())
 
-			testDynamicProvisioning(c, claim)
+			claim := newClaim(test, ns, "", true)
+			testDynamicProvisioning(test, c, claim, nil)
 		})
 	})
 })
 
-func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
+func newClaim(t storageClassTest, ns, suffix string, alpha bool) *api.PersistentVolumeClaim {
 	claim := api.PersistentVolumeClaim{
 		ObjectMeta: api.ObjectMeta{
 			GenerateName: "pvc-",
@@ -164,7 +368,7 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
 			},
 			Resources: api.ResourceRequirements{
 				Requests: api.ResourceList{
-					api.ResourceName(api.ResourceStorage): resource.MustParse(requestedSize),
+					api.ResourceName(api.ResourceStorage): resource.MustParse(t.claimSize),
 				},
 			},
 		},
@@ -176,9 +380,8 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
 		}
 	} else {
 		claim.Annotations = map[string]string{
-			"volume.beta.kubernetes.io/storage-class": "fast",
+			"volume.beta.kubernetes.io/storage-class": "myclass-" + suffix,
 		}
-
 	}
 
 	return &claim
@@ -224,32 +427,22 @@ func runInPodWithVolume(c *client.Client, ns, claimName, command string) {
 		},
 	}
 	pod, err := c.Pods(ns).Create(pod)
+	framework.ExpectNoError(err, "Failed to create pod: %v", err)
 	defer func() {
 		framework.ExpectNoError(c.Pods(ns).Delete(pod.Name, nil))
 	}()
-	framework.ExpectNoError(err, "Failed to create pod: %v", err)
 	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
 }
 
-func newStorageClass() *storage.StorageClass {
+func newStorageClass(t storageClassTest, suffix string) *storage.StorageClass {
-	var pluginName string
-
-	switch {
-	case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
-		pluginName = "kubernetes.io/gce-pd"
-	case framework.ProviderIs("aws"):
-		pluginName = "kubernetes.io/aws-ebs"
-	case framework.ProviderIs("openstack"):
-		pluginName = "kubernetes.io/cinder"
-	}
-
 	return &storage.StorageClass{
 		TypeMeta: unversioned.TypeMeta{
 			Kind: "StorageClass",
 		},
 		ObjectMeta: api.ObjectMeta{
-			Name: "fast",
+			Name: "myclass-" + suffix,
 		},
-		Provisioner: pluginName,
+		Provisioner: t.provisioner,
+		Parameters:  t.parameters,
 	}
 }