Merge pull request #27792 from jeffvance/e2e-prebind

Automatic merge from submit-queue

pv e2e refactor and pre-bind test

refactored persistentvolume e2e so that multiple It() tests can be run. Added one test case for pre-binding, but the overall structure of the test should allow additional test cases to be more easily added.
This commit is contained in:
Kubernetes Submit Queue 2016-08-04 05:26:43 -07:00 committed by GitHub
commit b9e808622d

View File

@ -18,10 +18,12 @@ package e2e
import (
"encoding/json"
"fmt"
"time"
. "github.com/onsi/ginkgo"
"k8s.io/kubernetes/pkg/api"
apierrs "k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/api/unversioned"
@ -29,148 +31,458 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
)
// Clean both server and client pods.
func persistentVolumeTestCleanup(client *client.Client, config VolumeTestConfig) {
// Delete the nfs-server pod.
func nfsServerPodCleanup(c *client.Client, config VolumeTestConfig) {
defer GinkgoRecover()
podClient := client.Pods(config.namespace)
podClient := c.Pods(config.namespace)
if config.serverImage != "" {
err := podClient.Delete(config.prefix+"-server", nil)
podName := config.prefix + "-server"
err := podClient.Delete(podName, nil)
if err != nil {
framework.Failf("Failed to delete the server pod: %v", err)
framework.Logf("Delete of %v pod failed: %v", podName, err)
}
}
}
func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) {
// Delete the PersistentVolume
framework.Logf("Deleting PersistentVolume")
// Delete the PV. Fail test if delete fails.
func deletePersistentVolume(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) {
if pv == nil {
return nil, fmt.Errorf("PV to be deleted is nil")
}
By("Deleting PersistentVolume")
framework.Logf("Deleting PersistentVolume %v", pv.Name)
err := c.PersistentVolumes().Delete(pv.Name)
if err != nil {
framework.Failf("Delete PersistentVolume failed: %v", err)
return pv, fmt.Errorf("Delete() PersistentVolume %v failed: %v", pv.Name, err)
}
// Wait for PersistentVolume to Delete
framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 30*time.Second)
// Wait for PersistentVolume to delete
deleteDuration := 90 * time.Second
err = framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, deleteDuration)
if err != nil {
return pv, fmt.Errorf("Unable to delete PersistentVolume %s after waiting for %v: %v", pv.Name, deleteDuration, err)
}
return nil, nil // success
}
// Delete the PVC and wait for the PV to become Available again.
// Validate that the PV has recycled (assumption here about reclaimPolicy).
func deletePVCandValidatePV(c *client.Client, ns string, pvc *api.PersistentVolumeClaim, pv *api.PersistentVolume) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) {
By("Deleting PersistentVolumeClaim to trigger PV Recycling")
framework.Logf("Deleting PersistentVolumeClaim %v to trigger PV Recycling", pvc.Name)
err := c.PersistentVolumeClaims(ns).Delete(pvc.Name)
if err != nil {
return pv, pvc, fmt.Errorf("Delete of PVC %v failed: %v", pvc.Name, err)
}
// Check that the PVC is really deleted.
pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name)
if err == nil {
return pv, pvc, fmt.Errorf("PVC %v deleted yet still exists", pvc.Name)
}
if !apierrs.IsNotFound(err) {
return pv, pvc, fmt.Errorf("Get on deleted PVC %v failed with error other than \"not found\": %v", pvc.Name, err)
}
// Wait for the PV's phase to return to Available
framework.Logf("Waiting for recycling process to complete.")
err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second)
if err != nil {
return pv, pvc, fmt.Errorf("Recycling failed: %v", err)
}
// Examine the pv.ClaimRef and UID. Expect nil values.
pv, err = c.PersistentVolumes().Get(pv.Name)
if err != nil {
return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name)
}
if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 {
crJSON, _ := json.Marshal(pv.Spec.ClaimRef)
return pv, pvc, fmt.Errorf("Expected PV %v's ClaimRef to be nil, or the claimRef's UID to be blank. Instead claimRef is: %v", pv.Name, string(crJSON))
}
return pv, pvc, nil
}
// create the PV resource. Fails test on error.
func createPV(c *client.Client, pv *api.PersistentVolume) (*api.PersistentVolume, error) {
pv, err := c.PersistentVolumes().Create(pv)
if err != nil {
return pv, fmt.Errorf("Create PersistentVolume %v failed: %v", pv.Name, err)
}
return pv, nil
}
// create the PVC resource. Fails test on error.
func createPVC(c *client.Client, ns string, pvc *api.PersistentVolumeClaim) (*api.PersistentVolumeClaim, error) {
pvc, err := c.PersistentVolumeClaims(ns).Create(pvc)
if err != nil {
return pvc, fmt.Errorf("Create PersistentVolumeClaim %v failed: %v", pvc.Name, err)
}
return pvc, nil
}
// Create a PV and PVC based on the passed in nfs-server ip and namespace.
// There are 4 combinations, 3 of which are supported here:
// 1) prebind and create pvc first
// 2) no prebind and create pvc first
// 3) no prebind and create pv first
// The case of prebinding and creating the pv first is not possible due to using a
// *generated* name in the pvc, and thus not knowing the claim's name until after
// it has been created.
// **Note: this function complements makePersistentVolume() and fills in the remaining
// name field in the pv's ClaimRef.
func createPVandPVC(c *client.Client, serverIP, ns string, pvFirst, preBind bool) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) {
var bindTo *api.PersistentVolumeClaim
var err error
pvc := makePersistentVolumeClaim(ns) // pvc.Name not known yet
bindTo = nil
if preBind { // implies pvc *must* be created before the pv
pvFirst = false
bindTo = pvc
}
pv := makePersistentVolume(serverIP, bindTo)
if pvFirst {
By("Creating the PV followed by the PVC")
pv, err = createPV(c, pv)
} else {
By("Creating the PVC followed by the PV")
pvc, err = createPVC(c, ns, pvc)
}
if err != nil {
return nil, nil, err
}
if pvFirst {
pvc, err = createPVC(c, ns, pvc)
if err != nil {
return pv, nil, err
}
} else {
// need to fill-in claimRef with pvc.Name
pv.Spec.ClaimRef.Name = pvc.Name
pv, err = createPV(c, pv)
if err != nil {
return nil, pvc, err
}
}
return pv, pvc, nil
}
// Wait for the pv and pvc to bind to each other. Fail test on errors.
func waitOnPVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) error {
// Wait for newly created PVC to bind to the PV
framework.Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name)
err := framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second)
if err != nil {
return fmt.Errorf("PersistentVolumeClaim failed to enter a bound state: %+v", err)
}
// Wait for PersistentVolume.Status.Phase to be Bound, which it should be
// since the PVC is already bound.
err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second)
if err != nil {
return fmt.Errorf("PersistentVolume failed to enter a bound state even though PVC is Bound: %+v", err)
}
return nil
}
// Waits for the pv and pvc to be bound to each other, then checks that the pv's
// claimRef matches the pvc. Fails test on errors.
func waitAndValidatePVandPVC(c *client.Client, ns string, pv *api.PersistentVolume, pvc *api.PersistentVolumeClaim) (*api.PersistentVolume, *api.PersistentVolumeClaim, error) {
var err error
// Wait for pv and pvc to bind to each other
if err = waitOnPVandPVC(c, ns, pv, pvc); err != nil {
return pv, pvc, err
}
// Check that the PersistentVolume.ClaimRef is valid and matches the PVC
framework.Logf("Checking PersistentVolume ClaimRef is non-nil")
pv, err = c.PersistentVolumes().Get(pv.Name)
if err != nil {
return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolume %v:", pv.Name)
}
pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name)
if err != nil {
return pv, pvc, fmt.Errorf("Cannot re-get PersistentVolumeClaim %v:", pvc.Name)
}
if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.UID != pvc.UID {
pvJSON, _ := json.Marshal(pv.Spec.ClaimRef)
return pv, pvc, fmt.Errorf("Expected Bound PersistentVolume %v to have valid ClaimRef: %+v", pv.Name, string(pvJSON))
}
return pv, pvc, nil
}
// Test the pod's exitcode to be zero.
func testPodSuccessOrFail(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error {
By("Pod should terminate with exitcode 0 (success)")
err := framework.WaitForPodSuccessInNamespace(c, pod.Name, pod.Spec.Containers[0].Name, ns)
if err != nil {
return fmt.Errorf("Pod %v returned non-zero exitcode: %+v", pod.Name, err)
}
framework.Logf("pod %v exited successfully", pod.Name)
return nil
}
// Delete the passed in pod.
func deletePod(f *framework.Framework, c *client.Client, ns string, pod *api.Pod) error {
framework.Logf("Deleting pod %v", pod.Name)
err := c.Pods(ns).Delete(pod.Name, nil)
if err != nil {
return fmt.Errorf("Pod %v encountered a delete error: %v", pod.Name, err)
}
// Wait for pod to terminate
err = f.WaitForPodTerminated(pod.Name, "")
if err != nil && !apierrs.IsNotFound(err) {
return fmt.Errorf("Pod %v will not teminate: %v", pod.Name, err)
}
// Re-get the pod to double check that it has been deleted; expect err
// Note: Get() writes a log error if the pod is not found
_, err = c.Pods(ns).Get(pod.Name)
if err == nil {
return fmt.Errorf("Pod %v has been deleted but able to re-Get the deleted pod", pod.Name)
}
if !apierrs.IsNotFound(err) {
return fmt.Errorf("Pod %v has been deleted but still exists: %v", pod.Name, err)
}
framework.Logf("Ignore \"not found\" error above. Pod %v successfully deleted", pod.Name)
return nil
}
// Create the test pod, wait for (hopefully) success, and then delete the pod.
func createWaitAndDeletePod(f *framework.Framework, c *client.Client, ns string, claimName string) error {
var errmsg string
framework.Logf("Creating nfs test pod")
// Make pod spec
pod := makeWritePod(ns, claimName)
// Instantiate pod (Create)
runPod, err := c.Pods(ns).Create(pod)
if err != nil || runPod == nil {
name := ""
if runPod != nil {
name = runPod.Name
}
return fmt.Errorf("Create test pod %v failed: %v", name, err)
}
// Wait for the test pod to complete its lifecycle
podErr := testPodSuccessOrFail(f, c, ns, runPod)
// Regardless of podErr above, delete the pod if it exists
if runPod != nil {
err = deletePod(f, c, ns, runPod)
}
// Check results of pod success and pod delete
if podErr != nil {
errmsg = fmt.Sprintf("Pod %v exited with non-zero exitcode: %v", runPod.Name, podErr)
}
if err != nil { // Delete error
if len(errmsg) > 0 {
errmsg += "; and "
}
errmsg += fmt.Sprintf("Delete error on pod %v: %v", runPod.Name, err)
}
if len(errmsg) > 0 {
return fmt.Errorf(errmsg)
}
return nil
}
var _ = framework.KubeDescribe("PersistentVolumes", func() {
// global vars for the It() tests below
f := framework.NewDefaultFramework("pv")
var c *client.Client
var ns string
var NFSconfig VolumeTestConfig
var serverIP string
var nfsServerPod *api.Pod
var pv *api.PersistentVolume
var pvc *api.PersistentVolumeClaim
var err error
// config for the nfs-server pod in the default namespace
NFSconfig = VolumeTestConfig{
namespace: api.NamespaceDefault,
prefix: "nfs",
serverImage: "gcr.io/google_containers/volume-nfs:0.6",
serverPorts: []int{2049},
}
BeforeEach(func() {
c = f.Client
ns = f.Namespace.Name
// If it doesn't exist, create the nfs server pod in "default" ns
// The "default" ns is used so that individual tests can delete
// their ns without impacting the nfs-server pod.
if nfsServerPod == nil {
nfsServerPod = startVolumeServer(c, NFSconfig)
serverIP = nfsServerPod.Status.PodIP
framework.Logf("NFS server IP address: %v", serverIP)
}
})
It("should create a PersistentVolume, Claim, and a client Pod that will test the read/write access of the volume[Flaky]", func() {
config := VolumeTestConfig{
namespace: ns,
prefix: "nfs",
serverImage: "gcr.io/google_containers/volume-nfs:0.6",
serverPorts: []int{2049},
AfterEach(func() {
if c != nil && len(ns) > 0 { // still have client and namespace
if pvc != nil && len(pvc.Name) > 0 {
// Delete the PersistentVolumeClaim
framework.Logf("AfterEach: PVC %v is non-nil, deleting claim", pvc.Name)
err := c.PersistentVolumeClaims(ns).Delete(pvc.Name)
if err != nil && !apierrs.IsNotFound(err) {
framework.Logf("AfterEach: delete of PersistentVolumeClaim %v error: %v", pvc.Name, err)
}
pvc = nil
}
if pv != nil && len(pv.Name) > 0 {
framework.Logf("AfterEach: PV %v is non-nil, deleting pv", pv.Name)
err := c.PersistentVolumes().Delete(pv.Name)
if err != nil && !apierrs.IsNotFound(err) {
framework.Logf("AfterEach: delete of PersistentVolume %v error: %v", pv.Name, err)
}
pv = nil
}
}
})
defer func() {
persistentVolumeTestCleanup(c, config)
}()
// Execute after *all* the tests have run
AddCleanupAction(func() {
if nfsServerPod != nil && c != nil {
framework.Logf("AfterSuite: nfs-server pod %v is non-nil, deleting pod", nfsServerPod.Name)
nfsServerPodCleanup(c, NFSconfig)
nfsServerPod = nil
}
})
// Create the nfs server pod
pod := startVolumeServer(c, config)
serverIP := pod.Status.PodIP
framework.Logf("NFS server IP address: %v", serverIP)
// Individual tests follow:
//
// Create an nfs PV, a claim that matches the PV, a pod that contains the
// claim. Verify that the PV and PVC bind correctly and that the pod can
// write to the nfs volume.
It("should create a PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() {
// Define the PersistentVolume and PersistentVolumeClaim
pv := makePersistentVolume(serverIP)
pvc := makePersistentVolumeClaim(ns)
// Create the PersistentVolume and wait for PersistentVolume.Status.Phase to be Available
// defer deletion to clean up the PV should the test fail post-creation.
framework.Logf("Creating PersistentVolume")
pv, err := c.PersistentVolumes().Create(pv)
pv, pvc, err = createPVandPVC(c, serverIP, ns, true /*pv first*/, false)
if err != nil {
framework.Failf("Create PersistentVolume failed: %v", err)
framework.Failf("%v", err)
}
defer deletePersistentVolume(c, pv)
framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 1*time.Second, 20*time.Second)
// Create the PersistentVolumeClaim and wait for Bound phase
framework.Logf("Creating PersistentVolumeClaim")
pvc, err = c.PersistentVolumeClaims(ns).Create(pvc)
pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc)
if err != nil {
framework.Failf("Create PersistentVolumeClaim failed: %v", err)
framework.Failf("%v", err)
}
framework.WaitForPersistentVolumeClaimPhase(api.ClaimBound, c, ns, pvc.Name, 3*time.Second, 300*time.Second)
// Wait for PersistentVolume.Status.Phase to be Bound. Can take several minutes.
err = framework.WaitForPersistentVolumePhase(api.VolumeBound, c, pv.Name, 3*time.Second, 300*time.Second)
By("Checking pod has write access to PersistentVolume")
if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil {
framework.Failf("%v", err)
}
// Delete the PVC before deleting PV, wait for PV to be Available
pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv)
if err != nil {
framework.Failf("PersistentVolume failed to enter a bound state: %+v", err)
}
// Check the PersistentVolume.ClaimRef.UID for non-nil value as confirmation of the bound state.
framework.Logf("Checking PersistentVolume ClaimRef is non-nil")
pv, err = c.PersistentVolumes().Get(pv.Name)
if pv.Spec.ClaimRef == nil || len(pv.Spec.ClaimRef.UID) == 0 {
pvJson, _ := json.MarshalIndent(pv, "", " ")
framework.Failf("Expected PersistentVolume to be bound, but got nil ClaimRef or UID: %+v", string(pvJson))
framework.Failf("%v", err)
}
// Check the PersistentVolumeClaim.Status.Phase for Bound state
framework.Logf("Checking PersistentVolumeClaim status is Bound")
pvc, err = c.PersistentVolumeClaims(ns).Get(pvc.Name)
if pvcPhase := pvc.Status.Phase; pvcPhase != "Bound" {
framework.Failf("Expected PersistentVolumeClaim status Bound. Actual: %+v. Error: %+v", pvcPhase, err)
}
// Check that the PersistentVolume's ClaimRef contains the UID of the PersistendVolumeClaim
if pvc.ObjectMeta.UID != pv.Spec.ClaimRef.UID {
framework.Failf("Binding failed: PersistentVolumeClaim UID does not match PersistentVolume's ClaimRef UID. ")
}
// writePod writes to the nfs volume
framework.Logf("Creating writePod")
pvc, _ = c.PersistentVolumeClaims(ns).Get(pvc.Name)
writePod := makeWritePod(ns, pvc.Name)
writePod, err = c.Pods(ns).Create(writePod)
// Last cleanup step is to delete the pv
pv, err = deletePersistentVolume(c, pv)
if err != nil {
framework.Failf("Create writePod failed: %+v", err)
framework.Failf("%v", err)
}
})
// Wait for the writePod to complete it's lifecycle
err = framework.WaitForPodSuccessInNamespace(c, writePod.Name, writePod.Spec.Containers[0].Name, writePod.Namespace)
// Create an nfs PV that is *pre-bound* to a claim. Create a pod that
// contains the claim. Verify that the PV and PVC bind correctly and that
// the pod can write to the nfs volume.
It("should create a pre-bound PersistentVolume, Claim, and Pod that will test write access of the volume [Flaky]", func() {
pv, pvc, err = createPVandPVC(c, serverIP, ns, false /*pvc first*/, true /*prebind*/)
if err != nil {
framework.Failf("WritePod exited with error: %+v", err)
} else {
framework.Logf("WritePod exited without error.")
framework.Failf("%v", err)
}
// Delete the PersistentVolumeClaim
framework.Logf("Deleting PersistentVolumeClaim to trigger PV Recycling")
err = c.PersistentVolumeClaims(ns).Delete(pvc.Name)
pv, pvc, err = waitAndValidatePVandPVC(c, ns, pv, pvc)
if err != nil {
framework.Failf("Delete PersistentVolumeClaim failed: %v", err)
framework.Failf("%v", err)
}
// Wait for the PersistentVolume phase to return to Available
framework.Logf("Waiting for recycling process to complete.")
err = framework.WaitForPersistentVolumePhase(api.VolumeAvailable, c, pv.Name, 3*time.Second, 300*time.Second)
// checkPod writes to the nfs volume
By("Checking pod has write access to pre-bound PersistentVolume")
// Instantiate pod, wait for it to exit, then delete it
if err = createWaitAndDeletePod(f, c, ns, pvc.Name); err != nil {
framework.Failf("%v", err)
}
// Delete the PVC before deleting PV, wait for PV to be Available
pv, pvc, err = deletePVCandValidatePV(c, ns, pvc, pv)
if err != nil {
framework.Failf("Recycling failed: %v", err)
framework.Failf("%v", err)
}
// Examine the PersistentVolume.ClaimRef and UID. Expect nil values.
pv, err = c.PersistentVolumes().Get(pv.Name)
if pv.Spec.ClaimRef != nil && len(pv.Spec.ClaimRef.UID) > 0 {
crjson, _ := json.MarshalIndent(pv.Spec.ClaimRef, "", " ")
framework.Failf("Expected a nil ClaimRef or UID. Found: ", string(crjson))
// Last cleanup step is to delete the pv
pv, err = deletePersistentVolume(c, pv)
if err != nil {
framework.Failf("%v", err)
}
})
})
func makePersistentVolume(serverIP string) *api.PersistentVolume {
// Takes an NFS server IP address and returns a PersistentVolume object for instantiation.
// Returns a PV definition based on the nfs server IP. If the PVC is not nil then the
// PV is defined with a ClaimRef which includes the PVC's namespace. If the PVC is
// nil then the PV is not defined with a ClaimRef.
// **Note: the passed-in claim does not have a name until it is created (instantiated)
// and thus the PV's ClaimRef cannot be completely filled-in in this func. Therefore,
// the ClaimRef's name is added later in createPVandPVC().
func makePersistentVolume(serverIP string, pvc *api.PersistentVolumeClaim) *api.PersistentVolume {
// Specs are expected to match this test's PersistentVolumeClaim
var claimRef *api.ObjectReference
claimRef = nil
if pvc != nil {
claimRef = &api.ObjectReference{
Name: pvc.Name,
Namespace: pvc.Namespace,
}
}
return &api.PersistentVolume{
ObjectMeta: api.ObjectMeta{
GenerateName: "nfs-",
@ -192,13 +504,15 @@ func makePersistentVolume(serverIP string) *api.PersistentVolume {
api.ReadOnlyMany,
api.ReadWriteMany,
},
ClaimRef: claimRef,
},
}
}
// Returns a PVC definition based on the namespace.
func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim {
// Takes a namespace and returns a PersistentVolumeClaim object for instantiation.
// Specs are expected to match this test's PersistentVolume
return &api.PersistentVolumeClaim{
ObjectMeta: api.ObjectMeta{
GenerateName: "pvc-",
@ -219,6 +533,8 @@ func makePersistentVolumeClaim(ns string) *api.PersistentVolumeClaim {
}
}
// Returns a pod definition based on the namespace. The pod references the PVC's
// name.
func makeWritePod(ns string, pvcName string) *api.Pod {
// Prepare pod that mounts the NFS volume again and
// checks that /mnt/index.html was scrubbed there