github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-08 03:33:56 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #32485 from jsafrane/e2e-provisioning-class

Browse Source 
Automatic merge from submit-queue

Add e2e tests for storageclass

- test pd-ssd and pd-standard on GCE,
- test all four volume types and encryption on AWS
- test just the default volume type on OpenStack (right now, there is no API
  to get list of them)

These tests are quite slow, e.g. there are two tests on AWS that has to run mkfs.ext4 on 500 GB magnetic drive with low IOPS, which takes ~3-4 minutes each.
This commit is contained in:
Kubernetes Submit Queue 2016-10-13 21:51:27 -07:00 committed by GitHub
commit 64f52a2725
2 changed files with 264 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/cloudprovider/providers/gce/gce.go
View File

@ -2562,6 +2562,21 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
return labels, nil
}
// TestDisk checks that a disk has given type. It should be used only for
// testing!
func (gce *GCECloud) TestDisk(diskName, volumeType string) error {
disk, err := gce.getDiskByNameUnknownZone(diskName)
if err != nil {
return err
}
if strings.HasSuffix(disk.Type, volumeType) {
return nil
}
return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
}
func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error {
instanceName := mapNodeNameToInstanceName(nodeName)
instance, err := gce.getInstanceByName(instanceName)
@ -2645,6 +2660,7 @@ func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, err
Zone: lastComponent(disk.Zone),
Name: disk.Name,
Kind: disk.Kind,
Type: disk.Type,
}
return d, nil
}
@ -2765,6 +2781,7 @@ type gceDisk struct {
Zone string
Name string
Kind string
Type string
}
// Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found

301
test/e2e/volume_provisioning.go
View File

@ -17,8 +17,13 @@ limitations under the License.
package e2e
import (
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/unversioned"
@ -30,16 +35,35 @@ import (
. "github.com/onsi/gomega"
)
const (
// Requested size of the volume
requestedSize = "1500Mi"
// Expected size of the volume is 2GiB, because all three supported cloud
// providers allocate volumes in 1GiB chunks.
expectedSize = "2Gi"
)
type storageClassTest struct {
name string
cloudProviders []string
provisioner string
parameters map[string]string
claimSize string
expectedSize string
pvCheck func(volume *api.PersistentVolume) error
}
func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeClaim) {
err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
func testDynamicProvisioning(t storageClassTest, client *client.Client, claim *api.PersistentVolumeClaim, class *storage.StorageClass) {
if class != nil {
By("creating a StorageClass " + class.Name)
class, err := client.Storage().StorageClasses().Create(class)
defer func() {
framework.Logf("deleting storage class %s", class.Name)
client.Storage().StorageClasses().Delete(class.Name)
}()
Expect(err).NotTo(HaveOccurred())
}
By("creating a claim")
claim, err := client.PersistentVolumeClaims(claim.Namespace).Create(claim)
defer func() {
framework.Logf("deleting claim %s/%s", claim.Namespace, claim.Name)
client.PersistentVolumeClaims(claim.Namespace).Delete(claim.Name)
}()
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
By("checking the claim")
@ -52,21 +76,28 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
Expect(err).NotTo(HaveOccurred())
// Check sizes
expectedCapacity := resource.MustParse(expectedSize)
expectedCapacity := resource.MustParse(t.expectedSize)
pvCapacity := pv.Spec.Capacity[api.ResourceName(api.ResourceStorage)]
Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()))
requestedCapacity := resource.MustParse(requestedSize)
requestedCapacity := resource.MustParse(t.claimSize)
claimCapacity := claim.Spec.Resources.Requests[api.ResourceName(api.ResourceStorage)]
Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()))
// Check PV properties
By("checking the PV")
Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(api.PersistentVolumeReclaimDelete))
expectedAccessModes := []api.PersistentVolumeAccessMode{api.ReadWriteOnce}
Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))
// Run the checker
if t.pvCheck != nil {
err = t.pvCheck(pv)
Expect(err).NotTo(HaveOccurred())
}
// We start two pods:
// - The first writes 'hello word' to the /mnt/test (= the volume).
// - The second one runs grep 'hello world' on /mnt/test.
@ -101,6 +132,54 @@ func testDynamicProvisioning(client *client.Client, claim *api.PersistentVolumeC
framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
}
// checkAWSEBS checks properties of an AWS EBS. Test framework does not
// instantiate full AWS provider, therefore we need use ec2 API directly.
func checkAWSEBS(volume *api.PersistentVolume, volumeType string, encrypted bool) error {
diskName := volume.Spec.AWSElasticBlockStore.VolumeID
client := ec2.New(session.New())
tokens := strings.Split(diskName, "/")
volumeID := tokens[len(tokens)-1]
request := &ec2.DescribeVolumesInput{
VolumeIds: []*string{&volumeID},
}
info, err := client.DescribeVolumes(request)
if err != nil {
return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
}
if len(info.Volumes) == 0 {
return fmt.Errorf("no volumes found for volume %q", volumeID)
}
if len(info.Volumes) > 1 {
return fmt.Errorf("multiple volumes found for volume %q", volumeID)
}
awsVolume := info.Volumes[0]
if awsVolume.VolumeType == nil {
return fmt.Errorf("expected volume type %q, got nil", volumeType)
}
if *awsVolume.VolumeType != volumeType {
return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
}
if encrypted && awsVolume.Encrypted == nil {
return fmt.Errorf("expected encrypted volume, got no encryption")
}
if encrypted && !*awsVolume.Encrypted {
return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
}
return nil
}
func checkGCEPD(volume *api.PersistentVolume, volumeType string) error {
cloud, err := getGCECloud()
if err != nil {
return err
}
diskName := volume.Spec.GCEPersistentDisk.PDName
return cloud.TestDisk(diskName, volumeType)
}
var _ = framework.KubeDescribe("Dynamic provisioning", func() {
f := framework.NewDefaultFramework("volume-provisioning")
@ -114,45 +193,170 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
})
framework.KubeDescribe("DynamicProvisioner", func() {
It("should create and delete persistent volumes [Slow]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
// This test checks that dynamic provisioning can provision a volume
// that can be used to persist data among pods.
By("creating a StorageClass")
class := newStorageClass()
_, err := c.Storage().StorageClasses().Create(class)
defer c.Storage().StorageClasses().Delete(class.Name)
Expect(err).NotTo(HaveOccurred())
tests := []storageClassTest{
{
"should provision SSD PD on GCE/GKE",
[]string{"gce", "gke"},
"kubernetes.io/gce-pd",
map[string]string{
"type": "pd-ssd",
// Check that GCE can parse "zone" parameter, however
// we can't create PDs in different than default zone
// as we don't know if we're running with Multizone=true
"zone": framework.TestContext.CloudConfig.Zone,
},
"1.5Gi",
"2Gi",
func(volume *api.PersistentVolume) error {
return checkGCEPD(volume, "pd-ssd")
},
},
{
"should provision HDD PD on GCE/GKE",
[]string{"gce", "gke"},
"kubernetes.io/gce-pd",
map[string]string{
"type": "pd-standard",
},
"1.5Gi",
"2Gi",
func(volume *api.PersistentVolume) error {
return checkGCEPD(volume, "pd-standard")
},
},
// AWS
{
"should provision gp2 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "gp2",
// Check that AWS can parse "zone" parameter, however
// we can't create PDs in different than default zone
// as we don't know zone names
"zone": framework.TestContext.CloudConfig.Zone,
},
"1.5Gi",
"2Gi",
func(volume *api.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", false)
},
},
{
"should provision io1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "io1",
"iopsPerGB": "50",
},
"3.5Gi",
"4Gi", // 4 GiB is minimum for io1
func(volume *api.PersistentVolume) error {
return checkAWSEBS(volume, "io1", false)
},
},
{
"should provision sc1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "sc1",
},
"500Gi", // minimum for sc1
"500Gi",
func(volume *api.PersistentVolume) error {
return checkAWSEBS(volume, "sc1", false)
},
},
{
"should provision st1 EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"type": "st1",
},
"500Gi", // minimum for st1
"500Gi",
func(volume *api.PersistentVolume) error {
return checkAWSEBS(volume, "st1", false)
},
},
{
"should provision encrypted EBS on AWS",
[]string{"aws"},
"kubernetes.io/aws-ebs",
map[string]string{
"encrypted": "true",
},
"1Gi",
"1Gi",
func(volume *api.PersistentVolume) error {
return checkAWSEBS(volume, "gp2", true)
},
},
// OpenStack generic tests (works on all OpenStack deployments)
{
"should provision generic Cinder volume on OpenStack",
[]string{"openstack"},
"kubernetes.io/cinder",
map[string]string{},
"1.5Gi",
"2Gi",
nil, // there is currently nothing to check on OpenStack
},
{
"should provision Cinder volume with empty volume type and zone on OpenStack",
[]string{"openstack"},
"kubernetes.io/cinder",
map[string]string{
"type": "",
"availability": "",
},
"1.5Gi",
"2Gi",
nil, // there is currently nothing to check on OpenStack
},
}
By("creating a claim with a dynamic provisioning annotation")
claim := newClaim(ns, false)
defer func() {
c.PersistentVolumeClaims(ns).Delete(claim.Name)
}()
claim, err = c.PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
for i, t := range tests {
// Beware of clojure, use local variables instead of those from
// outer scope
test := t
suffix := fmt.Sprintf("%d", i)
It(test.name, func() {
if len(t.cloudProviders) > 0 {
framework.SkipUnlessProviderIs(test.cloudProviders...)
}
testDynamicProvisioning(c, claim)
})
class := newStorageClass(test, suffix)
claim := newClaim(test, ns, suffix, false)
testDynamicProvisioning(test, c, claim, class)
})
}
})
framework.KubeDescribe("DynamicProvisioner Alpha", func() {
It("should create and delete alpha persistent volumes [Slow]", func() {
It("should provision alpha volumes [Slow]", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke")
By("creating a claim with an alpha dynamic provisioning annotation")
claim := newClaim(ns, true)
defer func() {
c.PersistentVolumeClaims(ns).Delete(claim.Name)
}()
claim, err := c.PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())
test := storageClassTest{
name: "alpha test",
claimSize: "1500Mi",
expectedSize: "2Gi",
}
testDynamicProvisioning(c, claim)
claim := newClaim(test, ns, "", true)
testDynamicProvisioning(test, c, claim, nil)
})
})
})
func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
func newClaim(t storageClassTest, ns, suffix string, alpha bool) *api.PersistentVolumeClaim {
claim := api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
GenerateName: "pvc-",
@ -164,7 +368,7 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse(requestedSize),
api.ResourceName(api.ResourceStorage): resource.MustParse(t.claimSize),
},
},
},
@ -176,9 +380,8 @@ func newClaim(ns string, alpha bool) *api.PersistentVolumeClaim {
}
} else {
claim.Annotations = map[string]string{
"volume.beta.kubernetes.io/storage-class": "fast",
"volume.beta.kubernetes.io/storage-class": "myclass-" + suffix,
}
}
return &claim
@ -224,32 +427,22 @@ func runInPodWithVolume(c *client.Client, ns, claimName, command string) {
},
}
pod, err := c.Pods(ns).Create(pod)
framework.ExpectNoError(err, "Failed to create pod: %v", err)
defer func() {
framework.ExpectNoError(c.Pods(ns).Delete(pod.Name, nil))
}()
framework.ExpectNoError(err, "Failed to create pod: %v", err)
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
}
func newStorageClass() *storage.StorageClass {
var pluginName string
switch {
case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
pluginName = "kubernetes.io/gce-pd"
case framework.ProviderIs("aws"):
pluginName = "kubernetes.io/aws-ebs"
case framework.ProviderIs("openstack"):
pluginName = "kubernetes.io/cinder"
}
func newStorageClass(t storageClassTest, suffix string) *storage.StorageClass {
return &storage.StorageClass{
TypeMeta: unversioned.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: api.ObjectMeta{
Name: "fast",
Name: "myclass-" + suffix,
},
Provisioner: pluginName,
Provisioner: t.provisioner,
Parameters: t.parameters,
}
}