diff --git a/test/e2e/persistent_volumes.go b/test/e2e/persistent_volumes.go index 7e9299a13b8..38bcfd69ce2 100644 --- a/test/e2e/persistent_volumes.go +++ b/test/e2e/persistent_volumes.go @@ -18,10 +18,12 @@ package e2e import ( "encoding/json" + "fmt" "time" . "github.com/onsi/ginkgo" "k8s.io/kubernetes/pkg/api" + apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/api/unversioned" @@ -29,148 +31,458 @@ import ( "k8s.io/kubernetes/test/e2e/framework" ) -// Clean both server and client pods. -func persistentVolumeTestCleanup(client *client.Client, config VolumeTestConfig) { +// Delete the nfs-server pod. +func nfsServerPodCleanup(c *client.Client, config VolumeTestConfig) { defer GinkgoRecover() - podClient := client.Pods(config.namespace) + podClient := c.Pods(config.namespace) if config.serverImage != "" { - err := podClient.Delete(config.prefix+"-server", nil) + podName := config.prefix + "-server" + err := podClient.Delete(podName, nil) if err != nil { - framework.Failf("Failed to delete the server pod: %v", err) + framework.Logf("Delete of %v pod failed: %v", podName, err) } } } -func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) { - // Delete the PersistentVolume - framework.Logf("Deleting PersistentVolume") +// Delete the PV. Fail test if delete fails. +func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) { + + if pv == nil { + return nil, fmt.Errorf("PV to be deleted is nil") + } + + By("Deleting PersistentVolume") + + framework.Logf("Deleting PersistentVolume %v", pv.Name) err := c.PersistentVolumes().Delete(pv.Name) if err != nil { - framework.Failf("Delete PersistentVolume failed: %v", err) + return pv, fmt.Errorf("Delete() PersistentVolume %v failed: %v", pv.Name, err) } - // Wait for PersistentVolume to Delete - framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 30*time.Second) + + // Wait for PersistentVolume to delete + deleteDuration := 90 * time.Second + err = framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, deleteDuration) + if err != nil { + return pv, fmt.Errorf("Unable to delete PersistentVolume %s after waiting for %v: %v", pv.Name, deleteDuration, err) + } + + return nil, nil // success +} + +// Delete the PVC and wait for the PV to become Available again. +// Validate that the PV has recycled (assumption here about reclaimPolicy). +func deletePVCandValidatePV(c *client.Client, ns string, pvc *api.PersistentVolumeClaim, pv *api.PersistentVolume) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { + + By("Deleting PersistentVolumeClaim to trigger PV Recycling") + + framework.Logf("Deleting PersistentVolumeClaim %v to trigger PV Recycling", pvc.Name) + err := c.PersistentVolumeClaims(ns).Delete(pvc.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Delete of PVC %v failed: %v", pvc.Name, err) + } + + // Check that the PVC is really deleted. + pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) + if err == nil { + return pv, pvc, fmt.Errorf("PVC %v deleted yet still exists", pvc.Name) + } + if !apierrs.IsNotFound(err) { + return pv, pvc, fmt.Errorf("Get on deleted PVC %v failed with error other than \"not found\": %v", pvc.Name, err) + } + + // Wait for the PV's phase to return to Available + framework.Logf("Waiting for recycling process to complete.") + err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second) + if err != nil { + return pv, pvc, fmt.Errorf("Recycling failed: %v", err) + } + + // Examine the pv.ClaimRef and UID. Expect nil values. + pv, err = c.PersistentVolumes().Get(pv.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name) + } + if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 { + crJSON, _ := json.Marshal(pv.Spec.ClaimRef) + return pv, pvc, fmt.Errorf("Expected PV %v's ClaimRef to be nil, or the claimRef's UID to be blank. Instead claimRef is: %v", pv.Name, string(crJSON)) + } + + return pv, pvc, nil +} + +// create the PV resource. Fails test on error. +func createPV(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) { + + pv, err := c.PersistentVolumes().Create(pv) + if err != nil { + return pv, fmt.Errorf("Create PersistentVolume %v failed: %v", pv.Name, err) + } + + return pv, nil +} + +// create the PVC resource. Fails test on error. +func createPVC(c *client.Client, ns string, pvc *api.PersistentVolumeClaim) (*api.PersistentVolumeClaim, error) { + + pvc, err := c.PersistentVolumeClaims(ns).Create(pvc) + if err != nil { + return pvc, fmt.Errorf("Create PersistentVolumeClaim %v failed: %v", pvc.Name, err) + } + + return pvc, nil +} + +// Create a PV and PVC based on the passed in nfs-server ip and namespace. +// There are 4 combinations, 3 of which are supported here: +// 1) prebind and create pvc first +// 2) no prebind and create pvc first +// 3) no prebind and create pv first +// The case of prebinding and creating the pv first is not possible due to using a +// *generated* name in the pvc, and thus not knowing the claim's name until after +// it has been created. +// **Note: this function complements makePersistentVolume() and fills in the remaining +// name field in the pv's ClaimRef. +func createPVandPVC(c *client.Client, serverIP, ns string, pvFirst, preBind bool) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { + + var bindTo *api.PersistentVolumeClaim + var err error + + pvc := makePersistentVolumeClaim(ns) // pvc.Name not known yet + + bindTo = nil + if preBind { // implies pvc *must* be created before the pv + pvFirst = false + bindTo = pvc + } + pv := makePersistentVolume(serverIP, bindTo) + + if pvFirst { + By("Creating the PV followed by the PVC") + pv, err = createPV(c, pv) + } else { + By("Creating the PVC followed by the PV") + pvc, err = createPVC(c, ns, pvc) + } + if err != nil { + return nil, nil, err + } + + if pvFirst { + pvc, err = createPVC(c, ns, pvc) + if err != nil { + return pv, nil, err + } + } else { + // need to fill-in claimRef with pvc.Name + pv.Spec.ClaimRef.Name = pvc.Name + pv, err = createPV(c, pv) + if err != nil { + return nil, pvc, err + } + } + + return pv, pvc, nil +} + +// Wait for the pv and pvc to bind to each other. Fail test on errors. +func waitOnPVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) error { + + // Wait for newly created PVC to bind to the PV + framework.Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name) + err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second) + if err != nil { + return fmt.Errorf("PersistentVolumeClaim failed to enter a bound state: %+v", err) + } + + // Wait for PersistentVolume.Status.Phase to be Bound, which it should be + // since the PVC is already bound. + err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second) + if err != nil { + return fmt.Errorf("PersistentVolume failed to enter a bound state even though PVC is Bound: %+v", err) + } + + return nil +} + +// Waits for the pv and pvc to be bound to each other, then checks that the pv's +// claimRef matches the pvc. Fails test on errors. +func waitAndValidatePVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) { + + var err error + + // Wait for pv and pvc to bind to each other + if err = waitOnPVandPVC(c, ns, pv, pvc); err != nil { + return pv, pvc, err + } + + // Check that the PersistentVolume.ClaimRef is valid and matches the PVC + framework.Logf("Checking PersistentVolume ClaimRef is non-nil") + pv, err = c.PersistentVolumes().Get(pv.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name) + } + + pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) + if err != nil { + return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolumeClaim %v:", pvc.Name) + } + + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.UID != pvc.UID { + pvJSON, _ := json.Marshal(pv.Spec.ClaimRef) + return pv, pvc, fmt.Errorf("Expected Bound PersistentVolume %v to have valid ClaimRef: %+v", pv.Name, string(pvJSON)) + } + + return pv, pvc, nil +} + +// Test the pod's exitcode to be zero. +func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error { + + By("Pod should terminate with exitcode 0 (success)") + + err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns) + if err != nil { + return fmt.Errorf("Pod %v returned non-zero exitcode: %+v", pod.Name, err) + } + + framework.Logf("pod %v exited successfully", pod.Name) + return nil +} + +// Delete the passed in pod. +func deletePod(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error { + + framework.Logf("Deleting pod %v", pod.Name) + err := c.Pods(ns).Delete(pod.Name, nil) + if err != nil { + return fmt.Errorf("Pod %v encountered a delete error: %v", pod.Name, err) + } + + // Wait for pod to terminate + err = f.WaitForPodTerminated(pod.Name, "") + if err != nil && !apierrs.IsNotFound(err) { + return fmt.Errorf("Pod %v will not teminate: %v", pod.Name, err) + } + + // Re-get the pod to double check that it has been deleted; expect err + // Note: Get() writes a log error if the pod is not found + _, err = c.Pods(ns).Get(pod.Name) + if err == nil { + return fmt.Errorf("Pod %v has been deleted but able to re-Get the deleted pod", pod.Name) + } + if !apierrs.IsNotFound(err) { + return fmt.Errorf("Pod %v has been deleted but still exists: %v", pod.Name, err) + } + + framework.Logf("Ignore \"not found\" error above. Pod %v successfully deleted", pod.Name) + return nil +} + +// Create the test pod, wait for (hopefully) success, and then delete the pod. +func createWaitAndDeletePod(f *framework.Framework, c *client.Client, ns string, claimName string) error { + + var errmsg string + + framework.Logf("Creating nfs test pod") + + // Make pod spec + pod := makeWritePod(ns, claimName) + + // Instantiate pod (Create) + runPod, err := c.Pods(ns).Create(pod) + if err != nil || runPod == nil { + name := "" + if runPod != nil { + name = runPod.Name + } + return fmt.Errorf("Create test pod %v failed: %v", name, err) + } + + // Wait for the test pod to complete its lifecycle + podErr := testPodSuccessOrFail(f, c, ns, runPod) + + // Regardless of podErr above, delete the pod if it exists + if runPod != nil { + err = deletePod(f, c, ns, runPod) + } + + // Check results of pod success and pod delete + if podErr != nil { + errmsg = fmt.Sprintf("Pod %v exited with non-zero exitcode: %v", runPod.Name, podErr) + } + if err != nil { // Delete error + if len(errmsg) > 0 { + errmsg += "; and " + } + errmsg += fmt.Sprintf("Delete error on pod %v: %v", runPod.Name, err) + } + + if len(errmsg) > 0 { + return fmt.Errorf(errmsg) + } + + return nil } var _ = framework.KubeDescribe("PersistentVolumes", func() { + + // global vars for the It() tests below f := framework.NewDefaultFramework("pv") var c *client.Client var ns string + var NFSconfig VolumeTestConfig + var serverIP string + var nfsServerPod *api.Pod + var pv *api.PersistentVolume + var pvc *api.PersistentVolumeClaim + var err error + + // config for the nfs-server pod in the default namespace + NFSconfig = VolumeTestConfig{ + namespace: api.NamespaceDefault, + prefix: "nfs", + serverImage: "gcr.io/google_containers/volume-nfs:0.6", + serverPorts: []int{2049}, + } + BeforeEach(func() { c = f.Client ns = f.Namespace.Name + + // If it doesn't exist, create the nfs server pod in "default" ns + // The "default" ns is used so that individual tests can delete + // their ns without impacting the nfs-server pod. + if nfsServerPod == nil { + nfsServerPod = startVolumeServer(c, NFSconfig) + serverIP = nfsServerPod.Status.PodIP + framework.Logf("NFS server IP address: %v", serverIP) + } }) - It("should create a PersistentVolume, Claim, and a client Pod that will test the read/write access of the volume[Flaky]", func() { - config := VolumeTestConfig{ - namespace: ns, - prefix: "nfs", - serverImage: "gcr.io/google_containers/volume-nfs:0.6", - serverPorts: []int{2049}, + AfterEach(func() { + if c != nil && len(ns) > 0 { // still have client and namespace + if pvc != nil && len(pvc.Name) > 0 { + // Delete the PersistentVolumeClaim + framework.Logf("AfterEach: PVC %v is non-nil, deleting claim", pvc.Name) + err := c.PersistentVolumeClaims(ns).Delete(pvc.Name) + if err != nil && !apierrs.IsNotFound(err) { + framework.Logf("AfterEach: delete of PersistentVolumeClaim %v error: %v", pvc.Name, err) + } + pvc = nil + } + if pv != nil && len(pv.Name) > 0 { + framework.Logf("AfterEach: PV %v is non-nil, deleting pv", pv.Name) + err := c.PersistentVolumes().Delete(pv.Name) + if err != nil && !apierrs.IsNotFound(err) { + framework.Logf("AfterEach: delete of PersistentVolume %v error: %v", pv.Name, err) + } + pv = nil + } } + }) - defer func() { - persistentVolumeTestCleanup(c, config) - }() + // Execute after *all* the tests have run + AddCleanupAction(func() { + if nfsServerPod != nil && c != nil { + framework.Logf("AfterSuite: nfs-server pod %v is non-nil, deleting pod", nfsServerPod.Name) + nfsServerPodCleanup(c, NFSconfig) + nfsServerPod = nil + } + }) - // Create the nfs server pod - pod := startVolumeServer(c, config) - serverIP := pod.Status.PodIP - framework.Logf("NFS server IP address: %v", serverIP) + // Individual tests follow: + // + // Create an nfs PV, a claim that matches the PV, a pod that contains the + // claim. Verify that the PV and PVC bind correctly and that the pod can + // write to the nfs volume. + It("should create a PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() { - // Define the PersistentVolume and PersistentVolumeClaim - pv := makePersistentVolume(serverIP) - pvc := makePersistentVolumeClaim(ns) - - // Create the PersistentVolume and wait for PersistentVolume.Status.Phase to be Available - // defer deletion to clean up the PV should the test fail post-creation. - framework.Logf("Creating PersistentVolume") - pv, err := c.PersistentVolumes().Create(pv) + pv, pvc, err = createPVandPVC(c, serverIP, ns, true /*pv first*/, false) if err != nil { - framework.Failf("Create PersistentVolume failed: %v", err) + framework.Failf("%v", err) } - defer deletePersistentVolume(c, pv) - framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 1*time.Second, 20*time.Second) - // Create the PersistentVolumeClaim and wait for Bound phase - framework.Logf("Creating PersistentVolumeClaim") - pvc, err = c.PersistentVolumeClaims(ns).Create(pvc) + pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc) if err != nil { - framework.Failf("Create PersistentVolumeClaim failed: %v", err) + framework.Failf("%v", err) } - framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second) - // Wait for PersistentVolume.Status.Phase to be Bound. Can take several minutes. - err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second) + By("Checking pod has write access to PersistentVolume") + + if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil { + framework.Failf("%v", err) + } + + // Delete the PVC before deleting PV, wait for PV to be Available + pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv) if err != nil { - framework.Failf("PersistentVolume failed to enter a bound state: %+v", err) - } - // Check the PersistentVolume.ClaimRef.UID for non-nil value as confirmation of the bound state. - framework.Logf("Checking PersistentVolume ClaimRef is non-nil") - pv, err = c.PersistentVolumes().Get(pv.Name) - if pv.Spec.ClaimRef == nil || len(pv.Spec.ClaimRef.UID) == 0 { - pvJson, _ := json.MarshalIndent(pv, "", " ") - framework.Failf("Expected PersistentVolume to be bound, but got nil ClaimRef or UID: %+v", string(pvJson)) + framework.Failf("%v", err) } - // Check the PersistentVolumeClaim.Status.Phase for Bound state - framework.Logf("Checking PersistentVolumeClaim status is Bound") - pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name) - if pvcPhase := pvc.Status.Phase; pvcPhase != "Bound" { - framework.Failf("Expected PersistentVolumeClaim status Bound. Actual: %+v. Error: %+v", pvcPhase, err) - } - - // Check that the PersistentVolume's ClaimRef contains the UID of the PersistendVolumeClaim - if pvc.ObjectMeta.UID != pv.Spec.ClaimRef.UID { - framework.Failf("Binding failed: PersistentVolumeClaim UID does not match PersistentVolume's ClaimRef UID. ") - } - - // writePod writes to the nfs volume - framework.Logf("Creating writePod") - pvc, _ = c.PersistentVolumeClaims(ns).Get(pvc.Name) - writePod := makeWritePod(ns, pvc.Name) - writePod, err = c.Pods(ns).Create(writePod) + // Last cleanup step is to delete the pv + pv, err = deletePersistentVolume(c, pv) if err != nil { - framework.Failf("Create writePod failed: %+v", err) + framework.Failf("%v", err) } + }) - // Wait for the writePod to complete it's lifecycle - err = framework.WaitForPodSuccessInNamespace(c, writePod.Name, writePod.Spec.Containers[0].Name, writePod.Namespace) + // Create an nfs PV that is *pre-bound* to a claim. Create a pod that + // contains the claim. Verify that the PV and PVC bind correctly and that + // the pod can write to the nfs volume. + It("should create a pre-bound PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() { + + pv, pvc, err = createPVandPVC(c, serverIP, ns, false /*pvc first*/, true /*prebind*/) if err != nil { - framework.Failf("WritePod exited with error: %+v", err) - } else { - framework.Logf("WritePod exited without error.") + framework.Failf("%v", err) } - // Delete the PersistentVolumeClaim - framework.Logf("Deleting PersistentVolumeClaim to trigger PV Recycling") - err = c.PersistentVolumeClaims(ns).Delete(pvc.Name) + pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc) if err != nil { - framework.Failf("Delete PersistentVolumeClaim failed: %v", err) + framework.Failf("%v", err) } - // Wait for the PersistentVolume phase to return to Available - framework.Logf("Waiting for recycling process to complete.") - err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second) + // checkPod writes to the nfs volume + By("Checking pod has write access to pre-bound PersistentVolume") + // Instantiate pod, wait for it to exit, then delete it + if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil { + framework.Failf("%v", err) + } + + // Delete the PVC before deleting PV, wait for PV to be Available + pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv) if err != nil { - framework.Failf("Recycling failed: %v", err) + framework.Failf("%v", err) } - // Examine the PersistentVolume.ClaimRef and UID. Expect nil values. - pv, err = c.PersistentVolumes().Get(pv.Name) - if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 { - crjson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ") - framework.Failf("Expected a nil ClaimRef or UID. Found: ", string(crjson)) + // Last cleanup step is to delete the pv + pv, err = deletePersistentVolume(c, pv) + if err != nil { + framework.Failf("%v", err) } - }) }) -func makePersistentVolume(serverIP string) *api.PersistentVolume { - // Takes an NFS server IP address and returns a PersistentVolume object for instantiation. +// Returns a PV definition based on the nfs server IP. If the PVC is not nil then the +// PV is defined with a ClaimRef which includes the PVC's namespace. If the PVC is +// nil then the PV is not defined with a ClaimRef. +// **Note: the passed-in claim does not have a name until it is created (instantiated) +// and thus the PV's ClaimRef cannot be completely filled-in in this func. Therefore, +// the ClaimRef's name is added later in createPVandPVC(). +func makePersistentVolume(serverIP string, pvc *api.PersistentVolumeClaim) *api.PersistentVolume { // Specs are expected to match this test's PersistentVolumeClaim + + var claimRef *api.ObjectReference + + claimRef = nil + if pvc != nil { + claimRef = &api.ObjectReference{ + Name: pvc.Name, + Namespace: pvc.Namespace, + } + } + return &api.PersistentVolume{ ObjectMeta: api.ObjectMeta{ GenerateName: "nfs-", @@ -192,13 +504,15 @@ func makePersistentVolume(serverIP string) *api.PersistentVolume { api.ReadOnlyMany, api.ReadWriteMany, }, + ClaimRef: claimRef, }, } } +// Returns a PVC definition based on the namespace. func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim { - // Takes a namespace and returns a PersistentVolumeClaim object for instantiation. // Specs are expected to match this test's PersistentVolume + return &api.PersistentVolumeClaim{ ObjectMeta: api.ObjectMeta{ GenerateName: "pvc-", @@ -219,6 +533,8 @@ func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim { } } +// Returns a pod definition based on the namespace. The pod references the PVC's +// name. func makeWritePod(ns string, pvcName string) *api.Pod { // Prepare pod that mounts the NFS volume again and // checks that /mnt/index.html was scrubbed there