Add e2e tests for storageclass

This reverts commit 22352d2844 and makes
gce.GetDiskByNameUnknownZone a public GCE cloud provider method.
This commit is contained in:
Jan Safranek 2017-04-05 11:49:49 +02:00
parent 8667d7c4f1
commit 67e1f2c08e
2 changed files with 359 additions and 128 deletions

View File

@ -77,10 +77,11 @@ type Disks interface {
// GCECloud implements Disks. // GCECloud implements Disks.
var _ Disks = (*GCECloud)(nil) var _ Disks = (*GCECloud)(nil)
type gceDisk struct { type GCEDisk struct {
Zone string Zone string
Name string Name string
Kind string Kind string
Type string
} }
func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error { func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error {
@ -259,7 +260,7 @@ func (gce *GCECloud) DeleteDisk(diskToDelete string) error {
// If zone is specified, the volume will only be found in the specified zone, // If zone is specified, the volume will only be found in the specified zone,
// otherwise all managed zones will be searched. // otherwise all managed zones will be searched.
func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]string, error) { func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]string, error) {
var disk *gceDisk var disk *GCEDisk
var err error var err error
if zone == "" { if zone == "" {
// We would like as far as possible to avoid this case, // We would like as far as possible to avoid this case,
@ -268,7 +269,7 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
// by name, so we have to continue to support that. // by name, so we have to continue to support that.
// However, wherever possible the zone should be passed (and it is passed // However, wherever possible the zone should be passed (and it is passed
// for most cases that we can control, e.g. dynamic volume provisioning) // for most cases that we can control, e.g. dynamic volume provisioning)
disk, err = gce.getDiskByNameUnknownZone(name) disk, err = gce.GetDiskByNameUnknownZone(name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -300,16 +301,17 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
return labels, nil return labels, nil
} }
// Returns a gceDisk for the disk, if it is found in the specified zone. // Returns a GCEDisk for the disk, if it is found in the specified zone.
// If not found, returns (nil, nil) // If not found, returns (nil, nil)
func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) { func (gce *GCECloud) findDiskByName(diskName string, zone string) (*GCEDisk, error) {
dc := contextWithNamespace(diskName, "gce_list_disk") dc := contextWithNamespace(diskName, "gce_list_disk")
disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Context(dc).Do() disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Context(dc).Do()
if err == nil { if err == nil {
d := &gceDisk{ d := &GCEDisk{
Zone: lastComponent(disk.Zone), Zone: lastComponent(disk.Zone),
Name: disk.Name, Name: disk.Name,
Kind: disk.Kind, Kind: disk.Kind,
Type: disk.Type,
} }
return d, nil return d, nil
} }
@ -320,7 +322,7 @@ func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, err
} }
// Like findDiskByName, but returns an error if the disk is not found // Like findDiskByName, but returns an error if the disk is not found
func (gce *GCECloud) getDiskByName(diskName string, zone string) (*gceDisk, error) { func (gce *GCECloud) getDiskByName(diskName string, zone string) (*GCEDisk, error) {
disk, err := gce.findDiskByName(diskName, zone) disk, err := gce.findDiskByName(diskName, zone)
if disk == nil && err == nil { if disk == nil && err == nil {
return nil, fmt.Errorf("GCE persistent disk not found: diskName=%q zone=%q", diskName, zone) return nil, fmt.Errorf("GCE persistent disk not found: diskName=%q zone=%q", diskName, zone)
@ -331,7 +333,7 @@ func (gce *GCECloud) getDiskByName(diskName string, zone string) (*gceDisk, erro
// Scans all managed zones to return the GCE PD // Scans all managed zones to return the GCE PD
// Prefer getDiskByName, if the zone can be established // Prefer getDiskByName, if the zone can be established
// Return cloudprovider.DiskNotFound if the given disk cannot be found in any zone // Return cloudprovider.DiskNotFound if the given disk cannot be found in any zone
func (gce *GCECloud) getDiskByNameUnknownZone(diskName string) (*gceDisk, error) { func (gce *GCECloud) GetDiskByNameUnknownZone(diskName string) (*GCEDisk, error) {
// Note: this is the gotcha right now with GCE PD support: // Note: this is the gotcha right now with GCE PD support:
// disk names are not unique per-region. // disk names are not unique per-region.
// (I can create two volumes with name "myvol" in e.g. us-central1-b & us-central1-f) // (I can create two volumes with name "myvol" in e.g. us-central1-b & us-central1-f)
@ -342,7 +344,7 @@ func (gce *GCECloud) getDiskByNameUnknownZone(diskName string) (*gceDisk, error)
// admission control, but that might be a little weird (values changing // admission control, but that might be a little weird (values changing
// on create) // on create)
var found *gceDisk var found *GCEDisk
for _, zone := range gce.managedZones { for _, zone := range gce.managedZones {
disk, err := gce.findDiskByName(diskName, zone) disk, err := gce.findDiskByName(diskName, zone)
if err != nil { if err != nil {
@ -383,7 +385,7 @@ func (gce *GCECloud) encodeDiskTags(tags map[string]string) (string, error) {
} }
func (gce *GCECloud) doDeleteDisk(diskToDelete string) error { func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
disk, err := gce.getDiskByNameUnknownZone(diskToDelete) disk, err := gce.GetDiskByNameUnknownZone(diskToDelete)
if err != nil { if err != nil {
return err return err
} }
@ -398,7 +400,7 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
} }
// Converts a Disk resource to an AttachedDisk resource. // Converts a Disk resource to an AttachedDisk resource.
func (gce *GCECloud) convertDiskToAttachedDisk(disk *gceDisk, readWrite string) *compute.AttachedDisk { func (gce *GCECloud) convertDiskToAttachedDisk(disk *GCEDisk, readWrite string) *compute.AttachedDisk {
return &compute.AttachedDisk{ return &compute.AttachedDisk{
DeviceName: disk.Name, DeviceName: disk.Name,
Kind: disk.Kind, Kind: disk.Kind,

View File

@ -18,15 +18,21 @@ package e2e
import ( import (
"fmt" "fmt"
"strings"
"time" "time"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
apierrs "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/sets"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/apiserver/pkg/authentication/serviceaccount"
"k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/v1"
@ -38,15 +44,40 @@ import (
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
) )
type storageClassTest struct {
name string
cloudProviders []string
provisioner string
parameters map[string]string
claimSize string
expectedSize string
pvCheck func(volume *v1.PersistentVolume) error
}
const ( const (
// Requested size of the volume
requestedSize = "1500Mi"
// Plugin name of the external provisioner // Plugin name of the external provisioner
externalPluginName = "example.com/nfs" externalPluginName = "example.com/nfs"
) )
func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, expectedSize string) { func testDynamicProvisioning(t storageClassTest, client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) {
err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) if class != nil {
By("creating a StorageClass " + class.Name)
class, err := client.Storage().StorageClasses().Create(class)
defer func() {
framework.Logf("deleting storage class %s", class.Name)
client.Storage().StorageClasses().Delete(class.Name, nil)
}()
Expect(err).NotTo(HaveOccurred())
}
By("creating a claim")
claim, err := client.Core().PersistentVolumeClaims(claim.Namespace).Create(claim)
defer func() {
framework.Logf("deleting claim %s/%s", claim.Namespace, claim.Name)
client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)
}()
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
By("checking the claim") By("checking the claim")
@ -59,21 +90,28 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Check sizes // Check sizes
expectedCapacity := resource.MustParse(expectedSize) expectedCapacity := resource.MustParse(t.expectedSize)
pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity") Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")
requestedCapacity := resource.MustParse(requestedSize) requestedCapacity := resource.MustParse(t.claimSize)
claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity") Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")
// Check PV properties // Check PV properties
By("checking the PV")
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete)) Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes)) Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name)) Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace)) Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
// Run the checker
if t.pvCheck != nil {
err = t.pvCheck(pv)
Expect(err).NotTo(HaveOccurred())
}
// We start two pods: // We start two pods:
// - The first writes 'hello word' to the /mnt/test (= the volume). // - The first writes 'hello word' to the /mnt/test (= the volume).
// - The second one runs grep 'hello world' on /mnt/test. // - The second one runs grep 'hello world' on /mnt/test.
@ -90,11 +128,68 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol
// Wait for the PV to get deleted. Technically, the first few delete // Wait for the PV to get deleted. Technically, the first few delete
// attempts may fail, as the volume is still attached to a node because // attempts may fail, as the volume is still attached to a node because
// kubelet is slowly cleaning up a pod, however it should succeed in a // kubelet is slowly cleaning up the previous pod, however it should succeed
// couple of minutes. Wait 20 minutes to recover from random cloud hiccups. // in a couple of minutes. Wait 20 minutes to recover from random cloud
// hiccups.
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute)) framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
} }
// checkAWSEBS checks properties of an AWS EBS. Test framework does not
// instantiate full AWS provider, therefore we need use ec2 API directly.
func checkAWSEBS(volume *v1.PersistentVolume, volumeType string, encrypted bool) error {
diskName := volume.Spec.AWSElasticBlockStore.VolumeID
client := ec2.New(session.New())
tokens := strings.Split(diskName, "/")
volumeID := tokens[len(tokens)-1]
request := &ec2.DescribeVolumesInput{
VolumeIds: []*string{&volumeID},
}
info, err := client.DescribeVolumes(request)
if err != nil {
return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
}
if len(info.Volumes) == 0 {
return fmt.Errorf("no volumes found for volume %q", volumeID)
}
if len(info.Volumes) > 1 {
return fmt.Errorf("multiple volumes found for volume %q", volumeID)
}
awsVolume := info.Volumes[0]
if awsVolume.VolumeType == nil {
return fmt.Errorf("expected volume type %q, got nil", volumeType)
}
if *awsVolume.VolumeType != volumeType {
return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
}
if encrypted && awsVolume.Encrypted == nil {
return fmt.Errorf("expected encrypted volume, got no encryption")
}
if encrypted && !*awsVolume.Encrypted {
return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
}
return nil
}
func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
cloud, err := framework.GetGCECloud()
if err != nil {
return err
}
diskName := volume.Spec.GCEPersistentDisk.PDName
disk, err := cloud.GetDiskByNameUnknownZone(diskName)
if err != nil {
return err
}
if !strings.HasSuffix(disk.Type, volumeType) {
return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
}
return nil
}
var _ = framework.KubeDescribe("Dynamic Provisioning", func() { var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
f := framework.NewDefaultFramework("volume-provisioning") f := framework.NewDefaultFramework("volume-provisioning")
@ -108,61 +203,182 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
}) })
framework.KubeDescribe("DynamicProvisioner", func() { framework.KubeDescribe("DynamicProvisioner", func() {
It("should create and delete persistent volumes [Slow] [Volume]", func() { // This test checks that dynamic provisioning can provision a volume
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke") // that can be used to persist data among pods.
By("creating a StorageClass") tests := []storageClassTest{
class := newStorageClass("", "internal") {
class, err := c.StorageV1().StorageClasses().Create(class) "should provision SSD PD on GCE/GKE",
defer c.StorageV1().StorageClasses().Delete(class.Name, nil) []string{"gce", "gke"},
Expect(err).NotTo(HaveOccurred()) "kubernetes.io/gce-pd",
map[string]string{
"type": "pd-ssd",
// Check that GCE can parse "zone" parameter, however
// we can't create PDs in different than default zone
// as we don't know if we're running with Multizone=true
"zone": framework.TestContext.CloudConfig.Zone,
},
"1.5Gi",
"2Gi",
func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-ssd")
},
},
{
"should provision HDD PD on GCE/GKE",
[]string{"gce", "gke"},
"kubernetes.io/gce-pd",
map[string]string{
"type": "pd-standard",
},
"1.5Gi",
"2Gi",
func(volume *v1.PersistentVolume) error {
return checkGCEPD(volume, "pd-standard")
},
},
// AWS
{
"should provision gp2 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "gp2",
// Check that AWS can parse "zone" parameter, however
// we can't create PDs in different than default zone
// as we don't know zone names
"zone": framework.TestContext.CloudConfig.Zone,
},
"1.5Gi",
"2Gi",
func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", false)
},
},
{
"should provision io1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "io1",
"iopsPerGB": "50",
},
"3.5Gi",
"4Gi", // 4 GiB is minimum for io1
func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "io1", false)
},
},
{
"should provision sc1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "sc1",
},
"500Gi", // minimum for sc1
"500Gi",
func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "sc1", false)
},
},
{
"should provision st1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "st1",
},
"500Gi", // minimum for st1
"500Gi",
func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "st1", false)
},
},
{
"should provision encrypted EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"encrypted": "true",
},
"1Gi",
"1Gi",
func(volume *v1.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", true)
},
},
// OpenStack generic tests (works on all OpenStack deployments)
{
"should provision generic Cinder volume on OpenStack",
[]string{"openstack"},
"kubernetes.io/cinder",
map[string]string{},
"1.5Gi",
"2Gi",
nil, // there is currently nothing to check on OpenStack
},
{
"should provision Cinder volume with empty volume type and zone on OpenStack",
[]string{"openstack"},
"kubernetes.io/cinder",
map[string]string{
"type": "",
"availability": "",
},
"1.5Gi",
"2Gi",
nil, // there is currently nothing to check on OpenStack
},
// vSphere generic test
{
"should provision generic vSphere",
[]string{"vsphere"},
"kubernetes.io/vsphere-volume",
map[string]string{},
"1.5Gi",
"1.5Gi",
nil,
},
}
By("creating a claim with a dynamic provisioning annotation") var betaTest *storageClassTest
claim := newClaim(ns) for i, t := range tests {
claim.Spec.StorageClassName = &class.Name // Beware of clojure, use local variables instead of those from
// outer scope
test := t
suffix := fmt.Sprintf("%d", i)
It(test.name+" [Slow] [Volume]", func() {
if len(t.cloudProviders) > 0 {
framework.SkipUnlessProviderIs(test.cloudProviders...)
}
defer func() { // Remember the last supported test for subsequent test of beta API
c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil) betaTest = &test
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
if framework.ProviderIs("vsphere") { class := newStorageClass(test, ns, suffix)
// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size claim := newClaim(test, ns, suffix)
// equal to requestedSize claim.Spec.StorageClassName = &class.Name
testDynamicProvisioning(c, claim, requestedSize) testDynamicProvisioning(test, c, claim, class)
} else { })
// Expected size of the volume is 2GiB, because the other three supported cloud }
// providers allocate volumes in 1GiB chunks.
testDynamicProvisioning(c, claim, "2Gi")
}
})
})
framework.KubeDescribe("DynamicProvisioner Beta", func() { // Run the last test with storage.k8s.io/v1beta1 and beta annotation on pvc
It("should create and delete persistent volumes [Slow] [Volume]", func() { if betaTest != nil {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke") It("Beta "+betaTest.name+" [Slow] [Volume]", func() {
class := newBetaStorageClass(*betaTest, "beta")
// we need to create the class manually, testDynamicProvisioning does not accept beta class
class, err := c.StorageV1beta1().StorageClasses().Create(class)
defer deleteStorageClass(c, class.Name)
Expect(err).NotTo(HaveOccurred())
By("creating a StorageClass") claim := newClaim(*betaTest, ns, "beta")
class := newBetaStorageClass("", "beta") claim.Annotations = map[string]string{
class, err := c.StorageV1beta1().StorageClasses().Create(class) v1.BetaStorageClassAnnotation: class.Name,
defer deleteStorageClass(c, class.Name) }
Expect(err).NotTo(HaveOccurred()) testDynamicProvisioning(*betaTest, c, claim, nil)
})
By("creating a claim with a dynamic provisioning annotation") }
claim := newClaim(ns)
claim.Annotations = map[string]string{
v1.BetaStorageClassAnnotation: class.Name,
}
defer func() {
framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
testDynamicProvisioning(c, claim, "2Gi")
})
// NOTE: Slow! The test will wait up to 5 minutes (framework.ClaimProvisionTimeout) when there is // NOTE: Slow! The test will wait up to 5 minutes (framework.ClaimProvisionTimeout) when there is
// no regression. // no regression.
@ -197,15 +413,19 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
} }
By("Creating a StorageClass for the unmanaged zone") By("Creating a StorageClass for the unmanaged zone")
sc := newBetaStorageClass("", suffix) test := storageClassTest{
// Set an unmanaged zone. name: "unmanaged_zone",
sc.Parameters = map[string]string{"zone": unmanagedZone} provisioner: "kubernetes.io/gce-pd",
sc, err = c.StorageV1beta1().StorageClasses().Create(sc) parameters: map[string]string{"zone": unmanagedZone},
claimSize: "1Gi",
}
sc := newStorageClass(test, ns, suffix)
sc, err = c.StorageV1().StorageClasses().Create(sc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer deleteStorageClass(c, sc.Name) defer deleteStorageClass(c, sc.Name)
By("Creating a claim and expecting it to timeout") By("Creating a claim and expecting it to timeout")
pvc := newClaim(ns) pvc := newClaim(test, ns, suffix)
pvc.Spec.StorageClassName = &sc.Name pvc.Spec.StorageClassName = &sc.Name
pvc, err = c.Core().PersistentVolumeClaims(ns).Create(pvc) pvc, err = c.Core().PersistentVolumeClaims(ns).Create(pvc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -223,21 +443,27 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
// not being deleted. // not being deleted.
// NOTE: Polls until no PVs are detected, times out at 5 minutes. // NOTE: Polls until no PVs are detected, times out at 5 minutes.
framework.SkipUnlessProviderIs("gce", "gke") framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
const raceAttempts int = 100 const raceAttempts int = 100
var residualPVs []*v1.PersistentVolume var residualPVs []*v1.PersistentVolume
By("Creating a StorageClass") By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
class := newBetaStorageClass("kubernetes.io/gce-pd", "") test := storageClassTest{
class, err := c.StorageV1beta1().StorageClasses().Create(class) name: "deletion race",
provisioner: "", // Use a native one based on current cloud provider
claimSize: "1Gi",
}
class := newStorageClass(test, ns, "race")
class, err := c.Storage().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
defer deleteStorageClass(c, class.Name) defer deleteStorageClass(c, class.Name)
framework.Logf("Created StorageClass %q", class.Name)
By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
claim := newClaim(ns)
// To increase chance of detection, attempt multiple iterations // To increase chance of detection, attempt multiple iterations
for i := 0; i < raceAttempts; i++ { for i := 0; i < raceAttempts; i++ {
suffix := fmt.Sprintf("race-%d", i)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
tmpClaim := framework.CreatePVC(c, ns, claim) tmpClaim := framework.CreatePVC(c, ns, claim)
framework.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns) framework.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns)
} }
@ -254,7 +480,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
if len(residualPVs) > 0 { if len(residualPVs) > 0 {
framework.Logf("Remaining PersistentVolumes:") framework.Logf("Remaining PersistentVolumes:")
for i, pv := range residualPVs { for i, pv := range residualPVs {
framework.Logf("%s%d) %s", "\t", i+1, pv.Name) framework.Logf("\t%d) %s", i+1, pv.Name)
} }
framework.Failf("Expected 0 PersistentVolumes remaining. Found %d", len(residualPVs)) framework.Failf("Expected 0 PersistentVolumes remaining. Found %d", len(residualPVs))
} }
@ -267,20 +493,17 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
By("creating a claim with an alpha dynamic provisioning annotation") By("creating a claim with an alpha dynamic provisioning annotation")
claim := newClaim(ns) test := storageClassTest{
claim.Annotations = map[string]string{v1.AlphaStorageClassAnnotation: ""} name: "alpha test",
claimSize: "2Gi",
defer func() { expectedSize: "2Gi",
framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}()
claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
if framework.ProviderIs("vsphere") {
testDynamicProvisioning(c, claim, requestedSize)
} else {
testDynamicProvisioning(c, claim, "2Gi")
} }
claim := newClaim(test, ns, "alpha")
claim.Annotations = map[string]string{
v1.AlphaStorageClassAnnotation: "true",
}
testDynamicProvisioning(test, c, claim, nil)
}) })
}) })
@ -301,29 +524,24 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
defer framework.DeletePodOrFail(c, ns, pod.Name) defer framework.DeletePodOrFail(c, ns, pod.Name)
By("creating a StorageClass") By("creating a StorageClass")
class := newStorageClass(externalPluginName, "external") test := storageClassTest{
class, err = c.StorageV1().StorageClasses().Create(class) name: "external provisioner test",
defer deleteStorageClass(c, class.Name) provisioner: externalPluginName,
Expect(err).NotTo(HaveOccurred()) claimSize: "1500Mi",
expectedSize: "1500Mi",
By("creating a claim with a dynamic provisioning annotation") }
claim := newClaim(ns) class := newStorageClass(test, ns, "external")
className := class.Name className := class.Name
claim := newClaim(test, ns, "external")
// the external provisioner understands Beta only right now, see // the external provisioner understands Beta only right now, see
// https://github.com/kubernetes-incubator/external-storage/issues/37 // https://github.com/kubernetes-incubator/external-storage/issues/37
// claim.Spec.StorageClassName = &className // claim.Spec.StorageClassName = &className
claim.Annotations = map[string]string{ claim.Annotations = map[string]string{
v1.BetaStorageClassAnnotation: className, v1.BetaStorageClassAnnotation: className,
} }
defer func() {
framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
// Expected size of the externally provisioned volume depends on the external By("creating a claim with a external provisioning annotation")
// provisioner: for nfs-provisioner used here, it's equal to requested testDynamicProvisioning(test, c, claim, class)
testDynamicProvisioning(c, claim, requestedSize)
}) })
}) })
@ -332,24 +550,26 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
By("creating a claim with no annotation") By("creating a claim with no annotation")
claim := newClaim(ns) test := storageClassTest{
name: "default",
claimSize: "2Gi",
expectedSize: "2Gi",
}
claim := newClaim(test, ns, "default")
defer func() { defer func() {
framework.DeletePersistentVolumeClaim(c, claim.Name, ns) framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}() }()
claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim) testDynamicProvisioning(test, c, claim, nil)
Expect(err).NotTo(HaveOccurred())
if framework.ProviderIs("vsphere") {
testDynamicProvisioning(c, claim, requestedSize)
} else {
testDynamicProvisioning(c, claim, "2Gi")
}
}) })
// Modifying the default storage class can be disruptive to other tests that depend on it // Modifying the default storage class can be disruptive to other tests that depend on it
It("should be disabled by changing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() { It("should be disabled by changing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
scName := getDefaultStorageClassName(c) scName := getDefaultStorageClassName(c)
test := storageClassTest{
name: "default",
claimSize: "2Gi",
}
By("setting the is-default StorageClass annotation to false") By("setting the is-default StorageClass annotation to false")
verifyDefaultStorageClass(c, scName, true) verifyDefaultStorageClass(c, scName, true)
@ -357,7 +577,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
updateDefaultStorageClass(c, scName, "false") updateDefaultStorageClass(c, scName, "false")
By("creating a claim with default storageclass and expecting it to timeout") By("creating a claim with default storageclass and expecting it to timeout")
claim := newClaim(ns) claim := newClaim(test, ns, "default")
defer func() { defer func() {
framework.DeletePersistentVolumeClaim(c, claim.Name, ns) framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}() }()
@ -377,6 +597,10 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
It("should be disabled by removing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() { It("should be disabled by removing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
scName := getDefaultStorageClassName(c) scName := getDefaultStorageClassName(c)
test := storageClassTest{
name: "default",
claimSize: "2Gi",
}
By("removing the is-default StorageClass annotation") By("removing the is-default StorageClass annotation")
verifyDefaultStorageClass(c, scName, true) verifyDefaultStorageClass(c, scName, true)
@ -384,7 +608,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
updateDefaultStorageClass(c, scName, "") updateDefaultStorageClass(c, scName, "")
By("creating a claim with default storageclass and expecting it to timeout") By("creating a claim with default storageclass and expecting it to timeout")
claim := newClaim(ns) claim := newClaim(test, ns, "default")
defer func() { defer func() {
framework.DeletePersistentVolumeClaim(c, claim.Name, ns) framework.DeletePersistentVolumeClaim(c, claim.Name, ns)
}() }()
@ -454,7 +678,7 @@ func updateDefaultStorageClass(c clientset.Interface, scName string, defaultStr
verifyDefaultStorageClass(c, scName, expectedDefault) verifyDefaultStorageClass(c, scName, expectedDefault)
} }
func newClaim(ns string) *v1.PersistentVolumeClaim { func newClaim(t storageClassTest, ns, suffix string) *v1.PersistentVolumeClaim {
claim := v1.PersistentVolumeClaim{ claim := v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-", GenerateName: "pvc-",
@ -466,7 +690,7 @@ func newClaim(ns string) *v1.PersistentVolumeClaim {
}, },
Resources: v1.ResourceRequirements{ Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{ Requests: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse(requestedSize), v1.ResourceName(v1.ResourceStorage): resource.MustParse(t.claimSize),
}, },
}, },
}, },
@ -536,28 +760,32 @@ func getDefaultPluginName() string {
return "" return ""
} }
func newStorageClass(pluginName, suffix string) *storage.StorageClass { func newStorageClass(t storageClassTest, ns string, suffix string) *storage.StorageClass {
pluginName := t.provisioner
if pluginName == "" { if pluginName == "" {
pluginName = getDefaultPluginName() pluginName = getDefaultPluginName()
} }
if suffix == "" { if suffix == "" {
suffix = "sc" suffix = "sc"
} }
return &storage.StorageClass{ return &storage.StorageClass{
TypeMeta: metav1.TypeMeta{ TypeMeta: metav1.TypeMeta{
Kind: "StorageClass", Kind: "StorageClass",
}, },
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
GenerateName: suffix + "-", // Name must be unique, so let's base it on namespace name
Name: ns + "-" + suffix,
}, },
Provisioner: pluginName, Provisioner: pluginName,
Parameters: t.parameters,
} }
} }
// TODO: remove when storage.k8s.io/v1beta1 and beta storage class annotations // TODO: remove when storage.k8s.io/v1beta1 and beta storage class annotations
// are removed. // are removed.
func newBetaStorageClass(pluginName, suffix string) *storagebeta.StorageClass { func newBetaStorageClass(t storageClassTest, suffix string) *storagebeta.StorageClass {
pluginName := t.provisioner
if pluginName == "" { if pluginName == "" {
pluginName = getDefaultPluginName() pluginName = getDefaultPluginName()
} }
@ -573,6 +801,7 @@ func newBetaStorageClass(pluginName, suffix string) *storagebeta.StorageClass {
GenerateName: suffix + "-", GenerateName: suffix + "-",
}, },
Provisioner: pluginName, Provisioner: pluginName,
Parameters: t.parameters,
} }
} }
@ -662,7 +891,7 @@ func waitForProvisionedVolumesDeleted(c clientset.Interface, scName string) ([]*
return true, err return true, err
} }
for _, pv := range allPVs.Items { for _, pv := range allPVs.Items {
if pv.Annotations[v1.BetaStorageClassAnnotation] == scName { if v1.GetPersistentVolumeClass(&pv) == scName {
remainingPVs = append(remainingPVs, &pv) remainingPVs = append(remainingPVs, &pv)
} }
} }