diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 30eb35a2794..aa1fe3c61c2 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -2562,6 +2562,21 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st return labels, nil } +// TestDisk checks that a disk has given type. It should be used only for +// testing! +func (gce *GCECloud) TestDisk(diskName, volumeType string) error { + disk, err := gce.getDiskByNameUnknownZone(diskName) + if err != nil { + return err + } + + if strings.HasSuffix(disk.Type, volumeType) { + return nil + } + + return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType) +} + func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error { instanceName := mapNodeNameToInstanceName(nodeName) instance, err := gce.getInstanceByName(instanceName) @@ -2645,6 +2660,7 @@ func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, err Zone: lastComponent(disk.Zone), Name: disk.Name, Kind: disk.Kind, + Type: disk.Type, } return d, nil } @@ -2765,6 +2781,7 @@ type gceDisk struct { Zone string Name string Kind string + Type string } // Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found diff --git a/test/e2e/volume_provisioning.go b/test/e2e/volume_provisioning.go index 3c37231210c..b9b064dd451 100644 --- a/test/e2e/volume_provisioning.go +++ b/test/e2e/volume_provisioning.go @@ -17,8 +17,13 @@ limitations under the License. package e2e import ( + "fmt" + "strings" "time" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/unversioned" @@ -30,16 +35,35 @@ import ( . "github.com/onsi/gomega" ) -const ( - // Requested size of the volume - requestedSize = "1500Mi" - // Expected size of the volume is 2GiB, because all three supported cloud - // providers allocate volumes in 1GiB chunks. - expectedSize = "2Gi" -) +type storageClassTest struct { + name string + cloudProviders []string + provisioner string + parameters map[string]string + claimSize string + expectedSize string + pvCheck func(volume *api.PersistentVolume) error +} -func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeClaim) { - err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) +func testDynamicProvisioning(t storageClassTest, client *client.Client, claim *api.PersistentVolumeClaim, class *storage.StorageClass) { + if class != nil { + By("creating a StorageClass " + class.Name) + class, err := client.Storage().StorageClasses().Create(class) + defer func() { + framework.Logf("deleting storage class %s", class.Name) + client.Storage().StorageClasses().Delete(class.Name) + }() + Expect(err).NotTo(HaveOccurred()) + } + + By("creating a claim") + claim, err := client.PersistentVolumeClaims(claim.Namespace).Create(claim) + defer func() { + framework.Logf("deleting claim %s/%s", claim.Namespace, claim.Name) + client.PersistentVolumeClaims(claim.Namespace).Delete(claim.Name) + }() + Expect(err).NotTo(HaveOccurred()) + err = framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) Expect(err).NotTo(HaveOccurred()) By("checking the claim") @@ -52,21 +76,28 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC Expect(err).NotTo(HaveOccurred()) // Check sizes - expectedCapacity := resource.MustParse(expectedSize) + expectedCapacity := resource.MustParse(t.expectedSize) pvCapacity := pv.Spec.Capacity[api.ResourceName(api.ResourceStorage)] Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value())) - requestedCapacity := resource.MustParse(requestedSize) + requestedCapacity := resource.MustParse(t.claimSize) claimCapacity := claim.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)] Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value())) // Check PV properties + By("checking the PV") Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(api.PersistentVolumeReclaimDelete)) expectedAccessModes := []api.PersistentVolumeAccessMode{api.ReadWriteOnce} Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes)) Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name)) Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace)) + // Run the checker + if t.pvCheck != nil { + err = t.pvCheck(pv) + Expect(err).NotTo(HaveOccurred()) + } + // We start two pods: // - The first writes 'hello word' to the /mnt/test (= the volume). // - The second one runs grep 'hello world' on /mnt/test. @@ -101,6 +132,54 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute)) } +// checkAWSEBS checks properties of an AWS EBS. Test framework does not +// instantiate full AWS provider, therefore we need use ec2 API directly. +func checkAWSEBS(volume *api.PersistentVolume, volumeType string, encrypted bool) error { + diskName := volume.Spec.AWSElasticBlockStore.VolumeID + + client := ec2.New(session.New()) + tokens := strings.Split(diskName, "/") + volumeID := tokens[len(tokens)-1] + + request := &ec2.DescribeVolumesInput{ + VolumeIds: []*string{&volumeID}, + } + info, err := client.DescribeVolumes(request) + if err != nil { + return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err) + } + if len(info.Volumes) == 0 { + return fmt.Errorf("no volumes found for volume %q", volumeID) + } + if len(info.Volumes) > 1 { + return fmt.Errorf("multiple volumes found for volume %q", volumeID) + } + + awsVolume := info.Volumes[0] + if awsVolume.VolumeType == nil { + return fmt.Errorf("expected volume type %q, got nil", volumeType) + } + if *awsVolume.VolumeType != volumeType { + return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType) + } + if encrypted && awsVolume.Encrypted == nil { + return fmt.Errorf("expected encrypted volume, got no encryption") + } + if encrypted && !*awsVolume.Encrypted { + return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted) + } + return nil +} + +func checkGCEPD(volume *api.PersistentVolume, volumeType string) error { + cloud, err := getGCECloud() + if err != nil { + return err + } + diskName := volume.Spec.GCEPersistentDisk.PDName + return cloud.TestDisk(diskName, volumeType) +} + var _ = framework.KubeDescribe("Dynamic provisioning", func() { f := framework.NewDefaultFramework("volume-provisioning") @@ -114,45 +193,170 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { }) framework.KubeDescribe("DynamicProvisioner", func() { - It("should create and delete persistent volumes [Slow]", func() { - framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke") + // This test checks that dynamic provisioning can provision a volume + // that can be used to persist data among pods. - By("creating a StorageClass") - class := newStorageClass() - _, err := c.Storage().StorageClasses().Create(class) - defer c.Storage().StorageClasses().Delete(class.Name) - Expect(err).NotTo(HaveOccurred()) + tests := []storageClassTest{ + { + "should provision SSD PD on GCE/GKE", + []string{"gce", "gke"}, + "kubernetes.io/gce-pd", + map[string]string{ + "type": "pd-ssd", + // Check that GCE can parse "zone" parameter, however + // we can't create PDs in different than default zone + // as we don't know if we're running with Multizone=true + "zone": framework.TestContext.CloudConfig.Zone, + }, + "1.5Gi", + "2Gi", + func(volume *api.PersistentVolume) error { + return checkGCEPD(volume, "pd-ssd") + }, + }, + { + "should provision HDD PD on GCE/GKE", + []string{"gce", "gke"}, + "kubernetes.io/gce-pd", + map[string]string{ + "type": "pd-standard", + }, + "1.5Gi", + "2Gi", + func(volume *api.PersistentVolume) error { + return checkGCEPD(volume, "pd-standard") + }, + }, + // AWS + { + "should provision gp2 EBS on AWS", + []string{"aws"}, + "kubernetes.io/aws-ebs", + map[string]string{ + "type": "gp2", + // Check that AWS can parse "zone" parameter, however + // we can't create PDs in different than default zone + // as we don't know zone names + "zone": framework.TestContext.CloudConfig.Zone, + }, + "1.5Gi", + "2Gi", + func(volume *api.PersistentVolume) error { + return checkAWSEBS(volume, "gp2", false) + }, + }, + { + "should provision io1 EBS on AWS", + []string{"aws"}, + "kubernetes.io/aws-ebs", + map[string]string{ + "type": "io1", + "iopsPerGB": "50", + }, + "3.5Gi", + "4Gi", // 4 GiB is minimum for io1 + func(volume *api.PersistentVolume) error { + return checkAWSEBS(volume, "io1", false) + }, + }, + { + "should provision sc1 EBS on AWS", + []string{"aws"}, + "kubernetes.io/aws-ebs", + map[string]string{ + "type": "sc1", + }, + "500Gi", // minimum for sc1 + "500Gi", + func(volume *api.PersistentVolume) error { + return checkAWSEBS(volume, "sc1", false) + }, + }, + { + "should provision st1 EBS on AWS", + []string{"aws"}, + "kubernetes.io/aws-ebs", + map[string]string{ + "type": "st1", + }, + "500Gi", // minimum for st1 + "500Gi", + func(volume *api.PersistentVolume) error { + return checkAWSEBS(volume, "st1", false) + }, + }, + { + "should provision encrypted EBS on AWS", + []string{"aws"}, + "kubernetes.io/aws-ebs", + map[string]string{ + "encrypted": "true", + }, + "1Gi", + "1Gi", + func(volume *api.PersistentVolume) error { + return checkAWSEBS(volume, "gp2", true) + }, + }, + // OpenStack generic tests (works on all OpenStack deployments) + { + "should provision generic Cinder volume on OpenStack", + []string{"openstack"}, + "kubernetes.io/cinder", + map[string]string{}, + "1.5Gi", + "2Gi", + nil, // there is currently nothing to check on OpenStack + }, + { + "should provision Cinder volume with empty volume type and zone on OpenStack", + []string{"openstack"}, + "kubernetes.io/cinder", + map[string]string{ + "type": "", + "availability": "", + }, + "1.5Gi", + "2Gi", + nil, // there is currently nothing to check on OpenStack + }, + } - By("creating a claim with a dynamic provisioning annotation") - claim := newClaim(ns, false) - defer func() { - c.PersistentVolumeClaims(ns).Delete(claim.Name) - }() - claim, err = c.PersistentVolumeClaims(ns).Create(claim) - Expect(err).NotTo(HaveOccurred()) + for i, t := range tests { + // Beware of clojure, use local variables instead of those from + // outer scope + test := t + suffix := fmt.Sprintf("%d", i) + It(test.name, func() { + if len(t.cloudProviders) > 0 { + framework.SkipUnlessProviderIs(test.cloudProviders...) + } - testDynamicProvisioning(c, claim) - }) + class := newStorageClass(test, suffix) + claim := newClaim(test, ns, suffix, false) + testDynamicProvisioning(test, c, claim, class) + }) + } }) framework.KubeDescribe("DynamicProvisioner Alpha", func() { - It("should create and delete alpha persistent volumes [Slow]", func() { + It("should provision alpha volumes [Slow]", func() { framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke") By("creating a claim with an alpha dynamic provisioning annotation") - claim := newClaim(ns, true) - defer func() { - c.PersistentVolumeClaims(ns).Delete(claim.Name) - }() - claim, err := c.PersistentVolumeClaims(ns).Create(claim) - Expect(err).NotTo(HaveOccurred()) + test := storageClassTest{ + name: "alpha test", + claimSize: "1500Mi", + expectedSize: "2Gi", + } - testDynamicProvisioning(c, claim) + claim := newClaim(test, ns, "", true) + testDynamicProvisioning(test, c, claim, nil) }) }) }) -func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim { +func newClaim(t storageClassTest, ns, suffix string, alpha bool) *api.PersistentVolumeClaim { claim := api.PersistentVolumeClaim{ ObjectMeta: api.ObjectMeta{ GenerateName: "pvc-", @@ -164,7 +368,7 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim { }, Resources: api.ResourceRequirements{ Requests: api.ResourceList{ - api.ResourceName(api.ResourceStorage): resource.MustParse(requestedSize), + api.ResourceName(api.ResourceStorage): resource.MustParse(t.claimSize), }, }, }, @@ -176,9 +380,8 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim { } } else { claim.Annotations = map[string]string{ - "volume.beta.kubernetes.io/storage-class": "fast", + "volume.beta.kubernetes.io/storage-class": "myclass-" + suffix, } - } return &claim @@ -224,32 +427,22 @@ func runInPodWithVolume(c *client.Client, ns, claimName, command string) { }, } pod, err := c.Pods(ns).Create(pod) + framework.ExpectNoError(err, "Failed to create pod: %v", err) defer func() { framework.ExpectNoError(c.Pods(ns).Delete(pod.Name, nil)) }() - framework.ExpectNoError(err, "Failed to create pod: %v", err) framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace)) } -func newStorageClass() *storage.StorageClass { - var pluginName string - - switch { - case framework.ProviderIs("gke"), framework.ProviderIs("gce"): - pluginName = "kubernetes.io/gce-pd" - case framework.ProviderIs("aws"): - pluginName = "kubernetes.io/aws-ebs" - case framework.ProviderIs("openstack"): - pluginName = "kubernetes.io/cinder" - } - +func newStorageClass(t storageClassTest, suffix string) *storage.StorageClass { return &storage.StorageClass{ TypeMeta: unversioned.TypeMeta{ Kind: "StorageClass", }, ObjectMeta: api.ObjectMeta{ - Name: "fast", + Name: "myclass-" + suffix, }, - Provisioner: pluginName, + Provisioner: t.provisioner, + Parameters: t.parameters, } }