From 9b28212eb699f63bc5ba6d7a253799da555e5b95 Mon Sep 17 00:00:00 2001 From: Jon Cope Date: Fri, 3 Jun 2016 15:06:05 -0500 Subject: [PATCH 1/2] Overhauled pv e2e test to reflect common lifecycle and de-flake --- test/e2e/persistent_volumes.go | 237 +++++++++++++++++++++++++-------- 1 file changed, 183 insertions(+), 54 deletions(-) diff --git a/test/e2e/persistent_volumes.go b/test/e2e/persistent_volumes.go index 7e9299a13b8..1d1bd991084 100644 --- a/test/e2e/persistent_volumes.go +++ b/test/e2e/persistent_volumes.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo" "k8s.io/kubernetes/pkg/api" + apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/api/unversioned" @@ -29,72 +30,144 @@ import ( "k8s.io/kubernetes/test/e2e/framework" ) -// Clean both server and client pods. -func persistentVolumeTestCleanup(client *client.Client, config VolumeTestConfig) { +// Delete the nfs-server pod. +func nfsServerPodCleanup(c *client.Client, config VolumeTestConfig) { defer GinkgoRecover() - podClient := client.Pods(config.namespace) + podClient := c.Pods(config.namespace) if config.serverImage != "" { - err := podClient.Delete(config.prefix+"-server", nil) + podName := config.prefix+"-server" + err := podClient.Delete(podName, nil) if err != nil { - framework.Failf("Failed to delete the server pod: %v", err) + framework.Failf("Delete of %v pod failed: %v", podName, err) } } } +// Delete the PV. Fail test if delete fails. func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) { // Delete the PersistentVolume framework.Logf("Deleting PersistentVolume") err := c.PersistentVolumes().Delete(pv.Name) if err != nil { - framework.Failf("Delete PersistentVolume failed: %v", err) + framework.Failf("Delete PersistentVolume %v failed: %v", pv.Name, err) } // Wait for PersistentVolume to Delete framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 30*time.Second) } +// Test the pod's exitcode to be zero, delete the pod, wait for it to be deleted, +// and fail if these steps return an error. +func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) { + + By("Pod should terminate with exitcode 0 (success)") + + err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns) + if err != nil { + framework.Failf("Pod %v returned non-zero exitcode: %+v", pod.Name, err) + } + + framework.Logf("Deleting pod %v after it exited successfully", pod.Name) + err = c.Pods(ns).Delete(pod.Name, nil) + if err != nil { + framework.Failf("Pod %v exited successfully but failed to delete: %+v", pod.Name, err) + } + + // Wait for pod to terminate + err = f.WaitForPodTerminated(pod.Name, "") + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("Pod %v has exitcode 0 but will not teminate: %v", pod.Name, err) + } + framework.Logf("Pod %v exited SUCCESSFULLY and was deleted", pod.Name) +} + + var _ = framework.KubeDescribe("PersistentVolumes", func() { f := framework.NewDefaultFramework("pv") var c *client.Client var ns string + var NFSconfig VolumeTestConfig + var serverIP string + var nfsServerPod *api.Pod + var checkPod *api.Pod + var pv *api.PersistentVolume + var pvc *api.PersistentVolumeClaim + var err error + + // config for the nfs-server pod in the default namespace + NFSconfig = VolumeTestConfig{ + namespace: api.NamespaceDefault, + prefix: "nfs", + serverImage: "gcr.io/google_containers/volume-nfs:0.6", + serverPorts: []int{2049}, + } + BeforeEach(func() { c = f.Client ns = f.Namespace.Name + + // If it doesn't exist, create the nfs server pod in "default" ns + if nfsServerPod == nil { + nfsServerPod = startVolumeServer(c, NFSconfig) + serverIP = nfsServerPod.Status.PodIP + framework.Logf("NFS server IP address: %v", serverIP) + } }) - It("should create a PersistentVolume, Claim, and a client Pod that will test the read/write access of the volume[Flaky]", func() { - config := VolumeTestConfig{ - namespace: ns, - prefix: "nfs", - serverImage: "gcr.io/google_containers/volume-nfs:0.6", - serverPorts: []int{2049}, + AfterEach(func() { + if c != nil && len(ns) > 0 { + if checkPod != nil { + // Wait for checkpod to complete termination + err = c.Pods(ns).Delete(checkPod.Name, nil) + if err != nil { + framework.Failf("AfterEach: pod %v delete ierror: %v", checkPod.Name, err) + } + checkPod = nil + } + + if pvc != nil { + // Delete the PersistentVolumeClaim + err = c.PersistentVolumeClaims(ns).Delete(pvc.Name) + if err != nil && !apierrs.IsNotFound(err) { + framework.Failf("AfterEach: delete of PersistentVolumeClaim %v experienced an unexpected error: %v", pvc.Name, err) + } + pvc = nil + } + if pv != nil { + deletePersistentVolume(c, pv) + pv = nil + } } + }) - defer func() { - persistentVolumeTestCleanup(c, config) - }() + // Execute after *all* the tests have run + AddCleanupAction(func() { + if nfsServerPod != nil && c != nil { + nfsServerPodCleanup(c, NFSconfig) + nfsServerPod = nil + } + }) - // Create the nfs server pod - pod := startVolumeServer(c, config) - serverIP := pod.Status.PodIP - framework.Logf("NFS server IP address: %v", serverIP) + + // Individual tests follow: + It("should create a PersistentVolume, Claim, and a client Pod that will test the read/write access of the volume", func() { // Define the PersistentVolume and PersistentVolumeClaim pv := makePersistentVolume(serverIP) pvc := makePersistentVolumeClaim(ns) // Create the PersistentVolume and wait for PersistentVolume.Status.Phase to be Available - // defer deletion to clean up the PV should the test fail post-creation. + By("Creating PV and PVC and waiting for Bound status") framework.Logf("Creating PersistentVolume") pv, err := c.PersistentVolumes().Create(pv) if err != nil { framework.Failf("Create PersistentVolume failed: %v", err) } - defer deletePersistentVolume(c, pv) + // Wait for PV to become Available. framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 1*time.Second, 20*time.Second) - // Create the PersistentVolumeClaim and wait for Bound phase + // Create the PersistentVolumeClaim and wait for Bound phase, can take several minutes. framework.Logf("Creating PersistentVolumeClaim") pvc, err = c.PersistentVolumeClaims(ns).Create(pvc) if err != nil { @@ -102,55 +175,46 @@ var _ = framework.KubeDescribe("PersistentVolumes", func() { } framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second) - // Wait for PersistentVolume.Status.Phase to be Bound. Can take several minutes. + // Wait for PersistentVolume.Status.Phase to be Bound, which it should already be since the PVC is bound. err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second) if err != nil { - framework.Failf("PersistentVolume failed to enter a bound state: %+v", err) + framework.Failf("PersistentVolume failed to enter a bound state even though PVC is Bound: %+v", err) } - // Check the PersistentVolume.ClaimRef.UID for non-nil value as confirmation of the bound state. + + // Check the PersistentVolume.ClaimRef is valid and matches the PVC framework.Logf("Checking PersistentVolume ClaimRef is non-nil") pv, err = c.PersistentVolumes().Get(pv.Name) - if pv.Spec.ClaimRef == nil || len(pv.Spec.ClaimRef.UID) == 0 { - pvJson, _ := json.MarshalIndent(pv, "", " ") - framework.Failf("Expected PersistentVolume to be bound, but got nil ClaimRef or UID: %+v", string(pvJson)) + if err != nil { + framework.Failf("Cannot re-get PersistentVolume %v:", pv.Name, err) } - - // Check the PersistentVolumeClaim.Status.Phase for Bound state - framework.Logf("Checking PersistentVolumeClaim status is Bound") pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) - if pvcPhase := pvc.Status.Phase; pvcPhase != "Bound" { - framework.Failf("Expected PersistentVolumeClaim status Bound. Actual: %+v. Error: %+v", pvcPhase, err) - } - - // Check that the PersistentVolume's ClaimRef contains the UID of the PersistendVolumeClaim - if pvc.ObjectMeta.UID != pv.Spec.ClaimRef.UID { - framework.Failf("Binding failed: PersistentVolumeClaim UID does not match PersistentVolume's ClaimRef UID. ") - } - - // writePod writes to the nfs volume - framework.Logf("Creating writePod") - pvc, _ = c.PersistentVolumeClaims(ns).Get(pvc.Name) - writePod := makeWritePod(ns, pvc.Name) - writePod, err = c.Pods(ns).Create(writePod) if err != nil { - framework.Failf("Create writePod failed: %+v", err) + framework.Failf("Cannot re-get PersistentVolumeClaim %v:", pvc.Name, err) + } + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.UID != pvc.UID { + pvJson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ") + framework.Failf("Expected Bound PersistentVolume %v to have valid ClaimRef: %+v", pv.Name, string(pvJson)) } - // Wait for the writePod to complete it's lifecycle - err = framework.WaitForPodSuccessInNamespace(c, writePod.Name, writePod.Spec.Containers[0].Name, writePod.Namespace) + // checkPod writes to the nfs volume + By("Checking pod has write access to PersistentVolume") + framework.Logf("Creating checkPod") + checkPod := makeWritePod(ns, pvc.Name) + checkPod, err = c.Pods(ns).Create(checkPod) if err != nil { - framework.Failf("WritePod exited with error: %+v", err) - } else { - framework.Logf("WritePod exited without error.") + framework.Failf("Create checkPod failed: %+v", err) } + // Wait for the checkPod to complete its lifecycle + testPodSuccessOrFail(f, c, ns, checkPod) + checkPod = nil // Delete the PersistentVolumeClaim + By("Deleting PersistentVolumeClaim to trigger PV Recycling") framework.Logf("Deleting PersistentVolumeClaim to trigger PV Recycling") err = c.PersistentVolumeClaims(ns).Delete(pvc.Name) if err != nil { framework.Failf("Delete PersistentVolumeClaim failed: %v", err) } - // Wait for the PersistentVolume phase to return to Available framework.Logf("Waiting for recycling process to complete.") err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second) @@ -158,13 +222,30 @@ var _ = framework.KubeDescribe("PersistentVolumes", func() { framework.Failf("Recycling failed: %v", err) } - // Examine the PersistentVolume.ClaimRef and UID. Expect nil values. + // Examine the PersistentVolume.ClaimRef and UID. Expect nil values. pv, err = c.PersistentVolumes().Get(pv.Name) if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 { crjson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ") - framework.Failf("Expected a nil ClaimRef or UID. Found: ", string(crjson)) + framework.Failf("Expected a nil pv.ClaimRef or empty UID. Found: ", string(crjson)) } + // Delete the PersistentVolume + By("Deleting PersistentVolume") + deletePersistentVolume(c, pv) + }) + + + It("should create another pod.... testing...", func() { + checkPod = makeTestPod(ns, serverIP) + checkPod, err = c.Pods(ns).Create(checkPod) + if err != nil { + framework.Failf("Error during testpod create: %v", err) + } + + // Wait for checkpod to complete it's lifecycle + testPodSuccessOrFail(f, c, ns, checkPod) + checkPod = nil // for AfterEach above + }) }) @@ -264,3 +345,51 @@ func makeWritePod(ns string, pvcName string) *api.Pod { }, } } + +func makeTestPod(ns string, nfsserver string) *api.Pod { + // Prepare pod that mounts the NFS volume again and + // checks that the volume can be written to via the mount. + + var isPrivileged bool = true + return &api.Pod{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Pod", + APIVersion: testapi.Default.GroupVersion().String(), + }, + ObjectMeta: api.ObjectMeta{ + GenerateName: "test-pod-", + Namespace: ns, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "test-pod", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "touch /mnt/FOO && exit 0 || exit 1"}, + VolumeMounts: []api.VolumeMount{ + { + Name: "nfs-vol", + MountPath: "/mnt", + }, + }, + SecurityContext: &api.SecurityContext{ + Privileged: &isPrivileged, + }, + }, + }, + Volumes: []api.Volume{ + { + Name: "nfs-vol", + VolumeSource: api.VolumeSource{ + NFS: &api.NFSVolumeSource{ + Server: nfsserver, + Path: "/", + }, + }, + }, + }, + }, + } +} + From 86578b2e2fd8a8bfcbb91fbc9c3b8353a7791955 Mon Sep 17 00:00:00 2001 From: Jeff Vance Date: Mon, 20 Jun 2016 20:54:11 -0700 Subject: [PATCH 2/2] refactored to support multiple It()s and added a prebinding test --- test/e2e/persistent_volumes.go | 535 ++++++++++++++++++++++----------- 1 file changed, 361 insertions(+), 174 deletions(-) diff --git a/test/e2e/persistent_volumes.go b/test/e2e/persistent_volumes.go index 1d1bd991084..38bcfd69ce2 100644 --- a/test/e2e/persistent_volumes.go +++ b/test/e2e/persistent_volumes.go @@ -18,6 +18,7 @@ package e2e import ( "encoding/json" + "fmt" "time" . "github.com/onsi/ginkgo" @@ -37,60 +38,303 @@ func nfsServerPodCleanup(c *client.Client, config VolumeTestConfig) { podClient := c.Pods(config.namespace) if config.serverImage != "" { - podName := config.prefix+"-server" + podName := config.prefix + "-server" err := podClient.Delete(podName, nil) if err != nil { - framework.Failf("Delete of %v pod failed: %v", podName, err) + framework.Logf("Delete of %v pod failed: %v", podName, err) } } } // Delete the PV. Fail test if delete fails. -func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) { - // Delete the PersistentVolume - framework.Logf("Deleting PersistentVolume") +func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) { + + if pv == nil { + return nil, fmt.Errorf("PV to be deleted is nil") + } + + By("Deleting PersistentVolume") + + framework.Logf("Deleting PersistentVolume %v", pv.Name) err := c.PersistentVolumes().Delete(pv.Name) if err != nil { - framework.Failf("Delete PersistentVolume %v failed: %v", pv.Name, err) + return pv, fmt.Errorf("Delete() PersistentVolume %v failed: %v", pv.Name, err) } - // Wait for PersistentVolume to Delete - framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 30*time.Second) + + // Wait for PersistentVolume to delete + deleteDuration := 90 * time.Second + err = framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, deleteDuration) + if err != nil { + return pv, fmt.Errorf("Unable to delete PersistentVolume %s after waiting for %v: %v", pv.Name, deleteDuration, err) + } + + return nil, nil // success } -// Test the pod's exitcode to be zero, delete the pod, wait for it to be deleted, -// and fail if these steps return an error. -func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) { +// Delete the PVC and wait for the PV to become Available again. +// Validate that the PV has recycled (assumption here about reclaimPolicy). +func deletePVCandValidatePV(c *client.Client, ns string, pvc *api.PersistentVolumeClaim, pv *api.PersistentVolume) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { - By("Pod should terminate with exitcode 0 (success)") + By("Deleting PersistentVolumeClaim to trigger PV Recycling") - err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns) - if err != nil { - framework.Failf("Pod %v returned non-zero exitcode: %+v", pod.Name, err) - } + framework.Logf("Deleting PersistentVolumeClaim %v to trigger PV Recycling", pvc.Name) + err := c.PersistentVolumeClaims(ns).Delete(pvc.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Delete of PVC %v failed: %v", pvc.Name, err) + } - framework.Logf("Deleting pod %v after it exited successfully", pod.Name) - err = c.Pods(ns).Delete(pod.Name, nil) - if err != nil { - framework.Failf("Pod %v exited successfully but failed to delete: %+v", pod.Name, err) - } + // Check that the PVC is really deleted. + pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) + if err == nil { + return pv, pvc, fmt.Errorf("PVC %v deleted yet still exists", pvc.Name) + } + if !apierrs.IsNotFound(err) { + return pv, pvc, fmt.Errorf("Get on deleted PVC %v failed with error other than \"not found\": %v", pvc.Name, err) + } - // Wait for pod to terminate - err = f.WaitForPodTerminated(pod.Name, "") - if err != nil && !apierrs.IsNotFound(err) { - framework.Failf("Pod %v has exitcode 0 but will not teminate: %v", pod.Name, err) - } - framework.Logf("Pod %v exited SUCCESSFULLY and was deleted", pod.Name) + // Wait for the PV's phase to return to Available + framework.Logf("Waiting for recycling process to complete.") + err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second) + if err != nil { + return pv, pvc, fmt.Errorf("Recycling failed: %v", err) + } + + // Examine the pv.ClaimRef and UID. Expect nil values. + pv, err = c.PersistentVolumes().Get(pv.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name) + } + if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 { + crJSON, _ := json.Marshal(pv.Spec.ClaimRef) + return pv, pvc, fmt.Errorf("Expected PV %v's ClaimRef to be nil, or the claimRef's UID to be blank. Instead claimRef is: %v", pv.Name, string(crJSON)) + } + + return pv, pvc, nil } +// create the PV resource. Fails test on error. +func createPV(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) { + + pv, err := c.PersistentVolumes().Create(pv) + if err != nil { + return pv, fmt.Errorf("Create PersistentVolume %v failed: %v", pv.Name, err) + } + + return pv, nil +} + +// create the PVC resource. Fails test on error. +func createPVC(c *client.Client, ns string, pvc *api.PersistentVolumeClaim) (*api.PersistentVolumeClaim, error) { + + pvc, err := c.PersistentVolumeClaims(ns).Create(pvc) + if err != nil { + return pvc, fmt.Errorf("Create PersistentVolumeClaim %v failed: %v", pvc.Name, err) + } + + return pvc, nil +} + +// Create a PV and PVC based on the passed in nfs-server ip and namespace. +// There are 4 combinations, 3 of which are supported here: +// 1) prebind and create pvc first +// 2) no prebind and create pvc first +// 3) no prebind and create pv first +// The case of prebinding and creating the pv first is not possible due to using a +// *generated* name in the pvc, and thus not knowing the claim's name until after +// it has been created. +// **Note: this function complements makePersistentVolume() and fills in the remaining +// name field in the pv's ClaimRef. +func createPVandPVC(c *client.Client, serverIP, ns string, pvFirst, preBind bool) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { + + var bindTo *api.PersistentVolumeClaim + var err error + + pvc := makePersistentVolumeClaim(ns) // pvc.Name not known yet + + bindTo = nil + if preBind { // implies pvc *must* be created before the pv + pvFirst = false + bindTo = pvc + } + pv := makePersistentVolume(serverIP, bindTo) + + if pvFirst { + By("Creating the PV followed by the PVC") + pv, err = createPV(c, pv) + } else { + By("Creating the PVC followed by the PV") + pvc, err = createPVC(c, ns, pvc) + } + if err != nil { + return nil, nil, err + } + + if pvFirst { + pvc, err = createPVC(c, ns, pvc) + if err != nil { + return pv, nil, err + } + } else { + // need to fill-in claimRef with pvc.Name + pv.Spec.ClaimRef.Name = pvc.Name + pv, err = createPV(c, pv) + if err != nil { + return nil, pvc, err + } + } + + return pv, pvc, nil +} + +// Wait for the pv and pvc to bind to each other. Fail test on errors. +func waitOnPVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) error { + + // Wait for newly created PVC to bind to the PV + framework.Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name) + err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second) + if err != nil { + return fmt.Errorf("PersistentVolumeClaim failed to enter a bound state: %+v", err) + } + + // Wait for PersistentVolume.Status.Phase to be Bound, which it should be + // since the PVC is already bound. + err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second) + if err != nil { + return fmt.Errorf("PersistentVolume failed to enter a bound state even though PVC is Bound: %+v", err) + } + + return nil +} + +// Waits for the pv and pvc to be bound to each other, then checks that the pv's +// claimRef matches the pvc. Fails test on errors. +func waitAndValidatePVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { + + var err error + + // Wait for pv and pvc to bind to each other + if err = waitOnPVandPVC(c, ns, pv, pvc); err != nil { + return pv, pvc, err + } + + // Check that the PersistentVolume.ClaimRef is valid and matches the PVC + framework.Logf("Checking PersistentVolume ClaimRef is non-nil") + pv, err = c.PersistentVolumes().Get(pv.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name) + } + + pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolumeClaim %v:", pvc.Name) + } + + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.UID != pvc.UID { + pvJSON, _ := json.Marshal(pv.Spec.ClaimRef) + return pv, pvc, fmt.Errorf("Expected Bound PersistentVolume %v to have valid ClaimRef: %+v", pv.Name, string(pvJSON)) + } + + return pv, pvc, nil +} + +// Test the pod's exitcode to be zero. +func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error { + + By("Pod should terminate with exitcode 0 (success)") + + err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns) + if err != nil { + return fmt.Errorf("Pod %v returned non-zero exitcode: %+v", pod.Name, err) + } + + framework.Logf("pod %v exited successfully", pod.Name) + return nil +} + +// Delete the passed in pod. +func deletePod(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error { + + framework.Logf("Deleting pod %v", pod.Name) + err := c.Pods(ns).Delete(pod.Name, nil) + if err != nil { + return fmt.Errorf("Pod %v encountered a delete error: %v", pod.Name, err) + } + + // Wait for pod to terminate + err = f.WaitForPodTerminated(pod.Name, "") + if err != nil && !apierrs.IsNotFound(err) { + return fmt.Errorf("Pod %v will not teminate: %v", pod.Name, err) + } + + // Re-get the pod to double check that it has been deleted; expect err + // Note: Get() writes a log error if the pod is not found + _, err = c.Pods(ns).Get(pod.Name) + if err == nil { + return fmt.Errorf("Pod %v has been deleted but able to re-Get the deleted pod", pod.Name) + } + if !apierrs.IsNotFound(err) { + return fmt.Errorf("Pod %v has been deleted but still exists: %v", pod.Name, err) + } + + framework.Logf("Ignore \"not found\" error above. Pod %v successfully deleted", pod.Name) + return nil +} + +// Create the test pod, wait for (hopefully) success, and then delete the pod. +func createWaitAndDeletePod(f *framework.Framework, c *client.Client, ns string, claimName string) error { + + var errmsg string + + framework.Logf("Creating nfs test pod") + + // Make pod spec + pod := makeWritePod(ns, claimName) + + // Instantiate pod (Create) + runPod, err := c.Pods(ns).Create(pod) + if err != nil || runPod == nil { + name := "" + if runPod != nil { + name = runPod.Name + } + return fmt.Errorf("Create test pod %v failed: %v", name, err) + } + + // Wait for the test pod to complete its lifecycle + podErr := testPodSuccessOrFail(f, c, ns, runPod) + + // Regardless of podErr above, delete the pod if it exists + if runPod != nil { + err = deletePod(f, c, ns, runPod) + } + + // Check results of pod success and pod delete + if podErr != nil { + errmsg = fmt.Sprintf("Pod %v exited with non-zero exitcode: %v", runPod.Name, podErr) + } + if err != nil { // Delete error + if len(errmsg) > 0 { + errmsg += "; and " + } + errmsg += fmt.Sprintf("Delete error on pod %v: %v", runPod.Name, err) + } + + if len(errmsg) > 0 { + return fmt.Errorf(errmsg) + } + + return nil +} var _ = framework.KubeDescribe("PersistentVolumes", func() { + + // global vars for the It() tests below f := framework.NewDefaultFramework("pv") var c *client.Client var ns string var NFSconfig VolumeTestConfig var serverIP string var nfsServerPod *api.Pod - var checkPod *api.Pod var pv *api.PersistentVolume var pvc *api.PersistentVolumeClaim var err error @@ -108,6 +352,8 @@ var _ = framework.KubeDescribe("PersistentVolumes", func() { ns = f.Namespace.Name // If it doesn't exist, create the nfs server pod in "default" ns + // The "default" ns is used so that individual tests can delete + // their ns without impacting the nfs-server pod. if nfsServerPod == nil { nfsServerPod = startVolumeServer(c, NFSconfig) serverIP = nfsServerPod.Status.PodIP @@ -116,26 +362,22 @@ var _ = framework.KubeDescribe("PersistentVolumes", func() { }) AfterEach(func() { - if c != nil && len(ns) > 0 { - if checkPod != nil { - // Wait for checkpod to complete termination - err = c.Pods(ns).Delete(checkPod.Name, nil) - if err != nil { - framework.Failf("AfterEach: pod %v delete ierror: %v", checkPod.Name, err) - } - checkPod = nil - } - - if pvc != nil { + if c != nil && len(ns) > 0 { // still have client and namespace + if pvc != nil && len(pvc.Name) > 0 { // Delete the PersistentVolumeClaim - err = c.PersistentVolumeClaims(ns).Delete(pvc.Name) + framework.Logf("AfterEach: PVC %v is non-nil, deleting claim", pvc.Name) + err := c.PersistentVolumeClaims(ns).Delete(pvc.Name) if err != nil && !apierrs.IsNotFound(err) { - framework.Failf("AfterEach: delete of PersistentVolumeClaim %v experienced an unexpected error: %v", pvc.Name, err) + framework.Logf("AfterEach: delete of PersistentVolumeClaim %v error: %v", pvc.Name, err) } pvc = nil } - if pv != nil { - deletePersistentVolume(c, pv) + if pv != nil && len(pv.Name) > 0 { + framework.Logf("AfterEach: PV %v is non-nil, deleting pv", pv.Name) + err := c.PersistentVolumes().Delete(pv.Name) + if err != nil && !apierrs.IsNotFound(err) { + framework.Logf("AfterEach: delete of PersistentVolume %v error: %v", pv.Name, err) + } pv = nil } } @@ -144,114 +386,103 @@ var _ = framework.KubeDescribe("PersistentVolumes", func() { // Execute after *all* the tests have run AddCleanupAction(func() { if nfsServerPod != nil && c != nil { + framework.Logf("AfterSuite: nfs-server pod %v is non-nil, deleting pod", nfsServerPod.Name) nfsServerPodCleanup(c, NFSconfig) nfsServerPod = nil } }) - // Individual tests follow: - It("should create a PersistentVolume, Claim, and a client Pod that will test the read/write access of the volume", func() { + // + // Create an nfs PV, a claim that matches the PV, a pod that contains the + // claim. Verify that the PV and PVC bind correctly and that the pod can + // write to the nfs volume. + It("should create a PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() { - // Define the PersistentVolume and PersistentVolumeClaim - pv := makePersistentVolume(serverIP) - pvc := makePersistentVolumeClaim(ns) - - // Create the PersistentVolume and wait for PersistentVolume.Status.Phase to be Available - By("Creating PV and PVC and waiting for Bound status") - framework.Logf("Creating PersistentVolume") - pv, err := c.PersistentVolumes().Create(pv) + pv, pvc, err = createPVandPVC(c, serverIP, ns, true /*pv first*/, false) if err != nil { - framework.Failf("Create PersistentVolume failed: %v", err) - } - // Wait for PV to become Available. - framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 1*time.Second, 20*time.Second) - - // Create the PersistentVolumeClaim and wait for Bound phase, can take several minutes. - framework.Logf("Creating PersistentVolumeClaim") - pvc, err = c.PersistentVolumeClaims(ns).Create(pvc) - if err != nil { - framework.Failf("Create PersistentVolumeClaim failed: %v", err) - } - framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second) - - // Wait for PersistentVolume.Status.Phase to be Bound, which it should already be since the PVC is bound. - err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second) - if err != nil { - framework.Failf("PersistentVolume failed to enter a bound state even though PVC is Bound: %+v", err) + framework.Failf("%v", err) } - // Check the PersistentVolume.ClaimRef is valid and matches the PVC - framework.Logf("Checking PersistentVolume ClaimRef is non-nil") - pv, err = c.PersistentVolumes().Get(pv.Name) + pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc) if err != nil { - framework.Failf("Cannot re-get PersistentVolume %v:", pv.Name, err) + framework.Failf("%v", err) } - pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) + + By("Checking pod has write access to PersistentVolume") + + if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil { + framework.Failf("%v", err) + } + + // Delete the PVC before deleting PV, wait for PV to be Available + pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv) if err != nil { - framework.Failf("Cannot re-get PersistentVolumeClaim %v:", pvc.Name, err) + framework.Failf("%v", err) } - if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.UID != pvc.UID { - pvJson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ") - framework.Failf("Expected Bound PersistentVolume %v to have valid ClaimRef: %+v", pv.Name, string(pvJson)) + + // Last cleanup step is to delete the pv + pv, err = deletePersistentVolume(c, pv) + if err != nil { + framework.Failf("%v", err) + } + }) + + // Create an nfs PV that is *pre-bound* to a claim. Create a pod that + // contains the claim. Verify that the PV and PVC bind correctly and that + // the pod can write to the nfs volume. + It("should create a pre-bound PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() { + + pv, pvc, err = createPVandPVC(c, serverIP, ns, false /*pvc first*/, true /*prebind*/) + if err != nil { + framework.Failf("%v", err) + } + + pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc) + if err != nil { + framework.Failf("%v", err) } // checkPod writes to the nfs volume - By("Checking pod has write access to PersistentVolume") - framework.Logf("Creating checkPod") - checkPod := makeWritePod(ns, pvc.Name) - checkPod, err = c.Pods(ns).Create(checkPod) + By("Checking pod has write access to pre-bound PersistentVolume") + // Instantiate pod, wait for it to exit, then delete it + if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil { + framework.Failf("%v", err) + } + + // Delete the PVC before deleting PV, wait for PV to be Available + pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv) if err != nil { - framework.Failf("Create checkPod failed: %+v", err) + framework.Failf("%v", err) } - // Wait for the checkPod to complete its lifecycle - testPodSuccessOrFail(f, c, ns, checkPod) - checkPod = nil - // Delete the PersistentVolumeClaim - By("Deleting PersistentVolumeClaim to trigger PV Recycling") - framework.Logf("Deleting PersistentVolumeClaim to trigger PV Recycling") - err = c.PersistentVolumeClaims(ns).Delete(pvc.Name) + // Last cleanup step is to delete the pv + pv, err = deletePersistentVolume(c, pv) if err != nil { - framework.Failf("Delete PersistentVolumeClaim failed: %v", err) + framework.Failf("%v", err) } - // Wait for the PersistentVolume phase to return to Available - framework.Logf("Waiting for recycling process to complete.") - err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second) - if err != nil { - framework.Failf("Recycling failed: %v", err) - } - - // Examine the PersistentVolume.ClaimRef and UID. Expect nil values. - pv, err = c.PersistentVolumes().Get(pv.Name) - if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 { - crjson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ") - framework.Failf("Expected a nil pv.ClaimRef or empty UID. Found: ", string(crjson)) - } - - // Delete the PersistentVolume - By("Deleting PersistentVolume") - deletePersistentVolume(c, pv) - }) - - - It("should create another pod.... testing...", func() { - checkPod = makeTestPod(ns, serverIP) - checkPod, err = c.Pods(ns).Create(checkPod) - if err != nil { - framework.Failf("Error during testpod create: %v", err) - } - - // Wait for checkpod to complete it's lifecycle - testPodSuccessOrFail(f, c, ns, checkPod) - checkPod = nil // for AfterEach above - }) }) -func makePersistentVolume(serverIP string) *api.PersistentVolume { - // Takes an NFS server IP address and returns a PersistentVolume object for instantiation. +// Returns a PV definition based on the nfs server IP. If the PVC is not nil then the +// PV is defined with a ClaimRef which includes the PVC's namespace. If the PVC is +// nil then the PV is not defined with a ClaimRef. +// **Note: the passed-in claim does not have a name until it is created (instantiated) +// and thus the PV's ClaimRef cannot be completely filled-in in this func. Therefore, +// the ClaimRef's name is added later in createPVandPVC(). +func makePersistentVolume(serverIP string, pvc *api.PersistentVolumeClaim) *api.PersistentVolume { // Specs are expected to match this test's PersistentVolumeClaim + + var claimRef *api.ObjectReference + + claimRef = nil + if pvc != nil { + claimRef = &api.ObjectReference{ + Name: pvc.Name, + Namespace: pvc.Namespace, + } + } + return &api.PersistentVolume{ ObjectMeta: api.ObjectMeta{ GenerateName: "nfs-", @@ -273,13 +504,15 @@ func makePersistentVolume(serverIP string) *api.PersistentVolume { api.ReadOnlyMany, api.ReadWriteMany, }, + ClaimRef: claimRef, }, } } +// Returns a PVC definition based on the namespace. func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim { - // Takes a namespace and returns a PersistentVolumeClaim object for instantiation. // Specs are expected to match this test's PersistentVolume + return &api.PersistentVolumeClaim{ ObjectMeta: api.ObjectMeta{ GenerateName: "pvc-", @@ -300,6 +533,8 @@ func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim { } } +// Returns a pod definition based on the namespace. The pod references the PVC's +// name. func makeWritePod(ns string, pvcName string) *api.Pod { // Prepare pod that mounts the NFS volume again and // checks that /mnt/index.html was scrubbed there @@ -345,51 +580,3 @@ func makeWritePod(ns string, pvcName string) *api.Pod { }, } } - -func makeTestPod(ns string, nfsserver string) *api.Pod { - // Prepare pod that mounts the NFS volume again and - // checks that the volume can be written to via the mount. - - var isPrivileged bool = true - return &api.Pod{ - TypeMeta: unversioned.TypeMeta{ - Kind: "Pod", - APIVersion: testapi.Default.GroupVersion().String(), - }, - ObjectMeta: api.ObjectMeta{ - GenerateName: "test-pod-", - Namespace: ns, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "test-pod", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/sh"}, - Args: []string{"-c", "touch /mnt/FOO && exit 0 || exit 1"}, - VolumeMounts: []api.VolumeMount{ - { - Name: "nfs-vol", - MountPath: "/mnt", - }, - }, - SecurityContext: &api.SecurityContext{ - Privileged: &isPrivileged, - }, - }, - }, - Volumes: []api.Volume{ - { - Name: "nfs-vol", - VolumeSource: api.VolumeSource{ - NFS: &api.NFSVolumeSource{ - Server: nfsserver, - Path: "/", - }, - }, - }, - }, - }, - } -} -