From 7184977c540428a911f981416e370df38069dd50 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Fri, 6 Jan 2017 17:07:34 -0500 Subject: [PATCH 1/2] Add e2e test for external pv provisioning --- test/e2e/volume_provisioning.go | 167 ++++++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 29 deletions(-) diff --git a/test/e2e/volume_provisioning.go b/test/e2e/volume_provisioning.go index 7c5425f9f31..ac728c66ee7 100644 --- a/test/e2e/volume_provisioning.go +++ b/test/e2e/volume_provisioning.go @@ -37,9 +37,14 @@ const ( // Expected size of the volume is 2GiB, because all three supported cloud // providers allocate volumes in 1GiB chunks. expectedSize = "2Gi" + // Expected size of the externally provisioned volume depends on the external + // provisioner: for nfs-provisioner used here, it's equal to requested + externalExpectedSize = "1500Mi" + // Plugin name of the external provisioner + externalPluginName = "example.com/nfs" ) -func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim) { +func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, external bool) { err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) Expect(err).NotTo(HaveOccurred()) @@ -54,6 +59,9 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol // Check sizes expectedCapacity := resource.MustParse(expectedSize) + if external { + expectedCapacity = resource.MustParse(externalExpectedSize) + } pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value())) @@ -79,21 +87,23 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol By("checking the created volume is readable and retains data") runInPodWithVolume(client, claim.Namespace, claim.Name, "grep 'hello world' /mnt/test/data") - // Ugly hack: if we delete the AWS/GCE/OpenStack volume here, it will - // probably collide with destruction of the pods above - the pods - // still have the volume attached (kubelet is slow...) and deletion - // of attached volume is not allowed by AWS/GCE/OpenStack. - // Kubernetes *will* retry deletion several times in - // pvclaimbinder-sync-period. - // So, technically, this sleep is not needed. On the other hand, - // the sync perion is 10 minutes and we really don't want to wait - // 10 minutes here. There is no way how to see if kubelet is - // finished with cleaning volumes. A small sleep here actually - // speeds up the test! - // Three minutes should be enough to clean up the pods properly. - // We've seen GCE PD detach to take more than 1 minute. - By("Sleeping to let kubelet destroy all pods") - time.Sleep(3 * time.Minute) + if !external { + // Ugly hack: if we delete the AWS/GCE/OpenStack volume here, it will + // probably collide with destruction of the pods above - the pods + // still have the volume attached (kubelet is slow...) and deletion + // of attached volume is not allowed by AWS/GCE/OpenStack. + // Kubernetes *will* retry deletion several times in + // pvclaimbinder-sync-period. + // So, technically, this sleep is not needed. On the other hand, + // the sync perion is 10 minutes and we really don't want to wait + // 10 minutes here. There is no way how to see if kubelet is + // finished with cleaning volumes. A small sleep here actually + // speeds up the test! + // Three minutes should be enough to clean up the pods properly. + // We've seen GCE PD detach to take more than 1 minute. + By("Sleeping to let kubelet destroy all pods") + time.Sleep(3 * time.Minute) + } By("deleting the claim") framework.ExpectNoError(client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)) @@ -119,7 +129,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke") By("creating a StorageClass") - class := newStorageClass() + class := newStorageClass("") _, err := c.Storage().StorageClasses().Create(class) defer c.Storage().StorageClasses().Delete(class.Name, nil) Expect(err).NotTo(HaveOccurred()) @@ -132,7 +142,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim) Expect(err).NotTo(HaveOccurred()) - testDynamicProvisioning(c, claim) + testDynamicProvisioning(c, claim, false) }) }) @@ -148,7 +158,31 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim) Expect(err).NotTo(HaveOccurred()) - testDynamicProvisioning(c, claim) + testDynamicProvisioning(c, claim, false) + }) + }) + + framework.KubeDescribe("DynamicProvisioner External", func() { + It("should let an external dynamic provisioner create and delete persistent volumes [Slow]", func() { + By("creating an external dynamic provisioner pod") + pod := startExternalProvisioner(c, ns) + defer c.Core().Pods(ns).Delete(pod.Name, nil) + + By("creating a StorageClass") + class := newStorageClass(externalPluginName) + _, err := c.Storage().StorageClasses().Create(class) + defer c.Storage().StorageClasses().Delete(class.Name, nil) + Expect(err).NotTo(HaveOccurred()) + + By("creating a claim with a dynamic provisioning annotation") + claim := newClaim(ns, false) + defer func() { + c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil) + }() + claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim) + Expect(err).NotTo(HaveOccurred()) + + testDynamicProvisioning(c, claim, true) }) }) }) @@ -232,16 +266,16 @@ func runInPodWithVolume(c clientset.Interface, ns, claimName, command string) { framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace)) } -func newStorageClass() *storage.StorageClass { - var pluginName string - - switch { - case framework.ProviderIs("gke"), framework.ProviderIs("gce"): - pluginName = "kubernetes.io/gce-pd" - case framework.ProviderIs("aws"): - pluginName = "kubernetes.io/aws-ebs" - case framework.ProviderIs("openstack"): - pluginName = "kubernetes.io/cinder" +func newStorageClass(pluginName string) *storage.StorageClass { + if pluginName == "" { + switch { + case framework.ProviderIs("gke"), framework.ProviderIs("gce"): + pluginName = "kubernetes.io/gce-pd" + case framework.ProviderIs("aws"): + pluginName = "kubernetes.io/aws-ebs" + case framework.ProviderIs("openstack"): + pluginName = "kubernetes.io/cinder" + } } return &storage.StorageClass{ @@ -254,3 +288,78 @@ func newStorageClass() *storage.StorageClass { Provisioner: pluginName, } } + +func startExternalProvisioner(c clientset.Interface, ns string) *v1.Pod { + podClient := c.Core().Pods(ns) + + provisionerPod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: v1.ObjectMeta{ + GenerateName: "external-provisioner-", + }, + + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nfs-provisioner", + Image: "quay.io/kubernetes_incubator/nfs-provisioner:v1.0.1", + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"DAC_READ_SEARCH"}, + }, + }, + Args: []string{ + "-provisioner=" + externalPluginName, + "-grace-period=0", + }, + Ports: []v1.ContainerPort{ + {Name: "nfs", ContainerPort: 2049}, + {Name: "mountd", ContainerPort: 20048}, + {Name: "rpcbind", ContainerPort: 111}, + {Name: "rpcbind-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP}, + }, + Env: []v1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + { + Name: "export-volume", + MountPath: "/export", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "export-volume", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + } + provisionerPod, err := podClient.Create(provisionerPod) + framework.ExpectNoError(err, "Failed to create %s pod: %v", provisionerPod.Name, err) + + framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, provisionerPod)) + + By("locating the provisioner pod") + pod, err := podClient.Get(provisionerPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Cannot locate the provisioner pod %v: %v", provisionerPod.Name, err) + + By("sleeping a bit to give the provisioner time to start") + time.Sleep(5 * time.Second) + return pod +} From 522b4b3f817cc8899b8e5a9786dcb4fb4c2857da Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Mon, 9 Jan 2017 12:16:33 -0500 Subject: [PATCH 2/2] Add expectedSize arg to testDynamicProvisioning --- test/e2e/volume_provisioning.go | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/test/e2e/volume_provisioning.go b/test/e2e/volume_provisioning.go index ac728c66ee7..35a4b027ef5 100644 --- a/test/e2e/volume_provisioning.go +++ b/test/e2e/volume_provisioning.go @@ -34,17 +34,11 @@ import ( const ( // Requested size of the volume requestedSize = "1500Mi" - // Expected size of the volume is 2GiB, because all three supported cloud - // providers allocate volumes in 1GiB chunks. - expectedSize = "2Gi" - // Expected size of the externally provisioned volume depends on the external - // provisioner: for nfs-provisioner used here, it's equal to requested - externalExpectedSize = "1500Mi" // Plugin name of the external provisioner externalPluginName = "example.com/nfs" ) -func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, external bool) { +func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, expectedSize string) { err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) Expect(err).NotTo(HaveOccurred()) @@ -59,9 +53,6 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol // Check sizes expectedCapacity := resource.MustParse(expectedSize) - if external { - expectedCapacity = resource.MustParse(externalExpectedSize) - } pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value())) @@ -87,23 +78,21 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol By("checking the created volume is readable and retains data") runInPodWithVolume(client, claim.Namespace, claim.Name, "grep 'hello world' /mnt/test/data") - if !external { - // Ugly hack: if we delete the AWS/GCE/OpenStack volume here, it will - // probably collide with destruction of the pods above - the pods - // still have the volume attached (kubelet is slow...) and deletion - // of attached volume is not allowed by AWS/GCE/OpenStack. - // Kubernetes *will* retry deletion several times in - // pvclaimbinder-sync-period. - // So, technically, this sleep is not needed. On the other hand, - // the sync perion is 10 minutes and we really don't want to wait - // 10 minutes here. There is no way how to see if kubelet is - // finished with cleaning volumes. A small sleep here actually - // speeds up the test! - // Three minutes should be enough to clean up the pods properly. - // We've seen GCE PD detach to take more than 1 minute. - By("Sleeping to let kubelet destroy all pods") - time.Sleep(3 * time.Minute) - } + // Ugly hack: if we delete the AWS/GCE/OpenStack volume here, it will + // probably collide with destruction of the pods above - the pods + // still have the volume attached (kubelet is slow...) and deletion + // of attached volume is not allowed by AWS/GCE/OpenStack. + // Kubernetes *will* retry deletion several times in + // pvclaimbinder-sync-period. + // So, technically, this sleep is not needed. On the other hand, + // the sync perion is 10 minutes and we really don't want to wait + // 10 minutes here. There is no way how to see if kubelet is + // finished with cleaning volumes. A small sleep here actually + // speeds up the test! + // Three minutes should be enough to clean up the pods properly. + // We've seen GCE PD detach to take more than 1 minute. + By("Sleeping to let kubelet destroy all pods") + time.Sleep(3 * time.Minute) By("deleting the claim") framework.ExpectNoError(client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil)) @@ -142,7 +131,9 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim) Expect(err).NotTo(HaveOccurred()) - testDynamicProvisioning(c, claim, false) + // Expected size of the volume is 2GiB, because all three supported cloud + // providers allocate volumes in 1GiB chunks. + testDynamicProvisioning(c, claim, "2Gi") }) }) @@ -158,7 +149,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim) Expect(err).NotTo(HaveOccurred()) - testDynamicProvisioning(c, claim, false) + testDynamicProvisioning(c, claim, "2Gi") }) }) @@ -182,7 +173,9 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() { claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim) Expect(err).NotTo(HaveOccurred()) - testDynamicProvisioning(c, claim, true) + // Expected size of the externally provisioned volume depends on the external + // provisioner: for nfs-provisioner used here, it's equal to requested + testDynamicProvisioning(c, claim, requestedSize) }) }) }) @@ -359,7 +352,5 @@ func startExternalProvisioner(c clientset.Interface, ns string) *v1.Pod { pod, err := podClient.Get(provisionerPod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Cannot locate the provisioner pod %v: %v", provisionerPod.Name, err) - By("sleeping a bit to give the provisioner time to start") - time.Sleep(5 * time.Second) return pod }