github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 13:37:30 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #43645 from copejon/encapsulate-e2e-pvs-with-selector-labels

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 38741, 41301, 43645, 43779, 42337)

De-Flake PersistentVolume E2E:  Isolate resources with selectors

PersistentVolume tests continue to flake because tests Claims often bind to other, parallel tests' volumes.  This is addressed by isolating test resource with selector labels specific to each test namespace.  Doing so enables deterministic binding within each test space and allows tests to run in parallel.

1.  Assign selector label to volumes and claims per test.
2. Remove `[Serial]` tag

cc @jeffvance 

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2017-03-29 14:53:33 -07:00 committed by GitHub
commit c0cd467676
6 changed files with 149 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
test/e2e/framework/pv_util.go
View File

@ -31,6 +31,7 @@ import (
apierrs "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
@ -44,6 +45,7 @@ import (
const ( const (
PDRetryTimeout = 5 * time.Minute PDRetryTimeout = 5 * time.Minute
PDRetryPollTime = 5 * time.Second PDRetryPollTime = 5 * time.Second
VolumeSelectorKey = "e2e-pv-pool"
) )
// Map of all PVs used in the multi pv-pvc tests. The key is the PV's name, which is // Map of all PVs used in the multi pv-pvc tests. The key is the PV's name, which is
@ -62,8 +64,10 @@ type PVMap map[string]pvval
type pvcval struct{} type pvcval struct{}
type PVCMap map[types.NamespacedName]pvcval type PVCMap map[types.NamespacedName]pvcval
// Configuration for a persistent volume. To create PVs for varying storage options (NFS, ceph, glusterFS, etc.) // PersistentVolumeConfig is consumed by MakePersistentVolume() to generate a PV object
// define the pvSource as below. prebind holds a pre-bound PVC if there is one. // for varying storage options (NFS, ceph, glusterFS, etc.).
// (+optional) prebind holds a pre-bound PVC
// Example pvSource:
// pvSource: api.PersistentVolumeSource{ // pvSource: api.PersistentVolumeSource{
// NFS: &api.NFSVolumeSource{ // NFS: &api.NFSVolumeSource{
// ... // ...
@ -74,6 +78,17 @@ type PersistentVolumeConfig struct {
Prebind *v1.PersistentVolumeClaim Prebind *v1.PersistentVolumeClaim
ReclaimPolicy v1.PersistentVolumeReclaimPolicy ReclaimPolicy v1.PersistentVolumeReclaimPolicy
NamePrefix string NamePrefix string
Labels labels.Set
}
// PersistentVolumeClaimConfig is consumed by MakePersistentVolumeClaim() to generate a PVC object.
// AccessModes defaults to all modes (RWO, RWX, ROX) if left empty
// (+optional) Annotations defines the PVC's annotations
type PersistentVolumeClaimConfig struct {
AccessModes []v1.PersistentVolumeAccessMode
Annotations map[string]string
Selector *metav1.LabelSelector
} }
// Clean up a pv and pvc in a single pv/pvc test case. // Clean up a pv and pvc in a single pv/pvc test case.
@ -185,7 +200,6 @@ func DeletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols PVMap,
// create the PV resource. Fails test on error. // create the PV resource. Fails test on error.
func createPV(c clientset.Interface, pv *v1.PersistentVolume) *v1.PersistentVolume { func createPV(c clientset.Interface, pv *v1.PersistentVolume) *v1.PersistentVolume {
pv, err := c.CoreV1().PersistentVolumes().Create(pv) pv, err := c.CoreV1().PersistentVolumes().Create(pv)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
return pv return pv
@ -193,7 +207,6 @@ func createPV(c clientset.Interface, pv *v1.PersistentVolume) *v1.PersistentVolu
// create the PVC resource. Fails test on error. // create the PVC resource. Fails test on error.
func CreatePVC(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { func CreatePVC(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Create(pvc) pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
return pvc return pvc
@ -205,18 +218,18 @@ func CreatePVC(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim)
// Note: in the pre-bind case the real PVC name, which is generated, is not // Note: in the pre-bind case the real PVC name, which is generated, is not
// known until after the PVC is instantiated. This is why the pvc is created // known until after the PVC is instantiated. This is why the pvc is created
// before the pv. // before the pv.
func CreatePVCPV(c clientset.Interface, pvConfig PersistentVolumeConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) { func CreatePVCPV(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
var preBindMsg string var preBindMsg string
// make the pvc definition first // make the pvc definition first
pvc := MakePersistentVolumeClaim(ns) pvc := MakePersistentVolumeClaim(pvcConfig, ns)
if preBind { if preBind {
preBindMsg = " pre-bound" preBindMsg = " pre-bound"
pvConfig.Prebind = pvc pvConfig.Prebind = pvc
} }
// make the pv spec // make the pv spec
pv := makePersistentVolume(pvConfig) pv := MakePersistentVolume(pvConfig)
By(fmt.Sprintf("Creating a PVC followed by a%s PV", preBindMsg)) By(fmt.Sprintf("Creating a PVC followed by a%s PV", preBindMsg))
// instantiate the pvc // instantiate the pvc
@ -238,7 +251,7 @@ func CreatePVCPV(c clientset.Interface, pvConfig PersistentVolumeConfig, ns stri
// Note: in the pre-bind case the real PV name, which is generated, is not // Note: in the pre-bind case the real PV name, which is generated, is not
// known until after the PV is instantiated. This is why the pv is created // known until after the PV is instantiated. This is why the pv is created
// before the pvc. // before the pvc.
func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) { func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
preBindMsg := "" preBindMsg := ""
if preBind { if preBind {
@ -247,8 +260,8 @@ func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, ns stri
Logf("Creating a PV followed by a%s PVC", preBindMsg) Logf("Creating a PV followed by a%s PVC", preBindMsg)
// make the pv and pvc definitions // make the pv and pvc definitions
pv := makePersistentVolume(pvConfig) pv := MakePersistentVolume(pvConfig)
pvc := MakePersistentVolumeClaim(ns) pvc := MakePersistentVolumeClaim(pvcConfig, ns)
// instantiate the pv // instantiate the pv
pv = createPV(c, pv) pv = createPV(c, pv)
@ -264,7 +277,7 @@ func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, ns stri
// Create the desired number of PVs and PVCs and return them in separate maps. If the // Create the desired number of PVs and PVCs and return them in separate maps. If the
// number of PVs != the number of PVCs then the min of those two counts is the number of // number of PVs != the number of PVCs then the min of those two counts is the number of
// PVs expected to bind. // PVs expected to bind.
func CreatePVsPVCs(numpvs, numpvcs int, c clientset.Interface, ns string, pvConfig PersistentVolumeConfig) (PVMap, PVCMap) { func CreatePVsPVCs(numpvs, numpvcs int, c clientset.Interface, ns string, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig) (PVMap, PVCMap) {
var i int var i int
var pv *v1.PersistentVolume var pv *v1.PersistentVolume
@ -282,19 +295,19 @@ func CreatePVsPVCs(numpvs, numpvcs int, c clientset.Interface, ns string, pvConf
// create pvs and pvcs // create pvs and pvcs
for i = 0; i < pvsToCreate; i++ { for i = 0; i < pvsToCreate; i++ {
pv, pvc = CreatePVPVC(c, pvConfig, ns, false) pv, pvc = CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
pvMap[pv.Name] = pvval{} pvMap[pv.Name] = pvval{}
pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{} pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
} }
// create extra pvs or pvcs as needed // create extra pvs or pvcs as needed
for i = 0; i < extraPVs; i++ { for i = 0; i < extraPVs; i++ {
pv = makePersistentVolume(pvConfig) pv = MakePersistentVolume(pvConfig)
pv = createPV(c, pv) pv = createPV(c, pv)
pvMap[pv.Name] = pvval{} pvMap[pv.Name] = pvval{}
} }
for i = 0; i < extraPVCs; i++ { for i = 0; i < extraPVCs; i++ {
pvc = MakePersistentVolumeClaim(ns) pvc = MakePersistentVolumeClaim(pvcConfig, ns)
pvc = CreatePVC(c, ns, pvc) pvc = CreatePVC(c, ns, pvc)
pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{} pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
} }
@ -361,7 +374,7 @@ func WaitAndVerifyBinds(c clientset.Interface, ns string, pvols PVMap, claims PV
// indicate non-test PVC interference or a bug in the test // indicate non-test PVC interference or a bug in the test
pvcKey := makePvcKey(ns, cr.Name) pvcKey := makePvcKey(ns, cr.Name)
_, found := claims[pvcKey] _, found := claims[pvcKey]
Expect(found).To(BeTrue()) Expect(found).To(BeTrue(), fmt.Sprintf("PersistentVolume (%q) ClaimRef (%q) does not match any test claims.", pv.Name, cr.Name))
err = WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, cr.Name, 3*time.Second, 180*time.Second) err = WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, cr.Name, 3*time.Second, 180*time.Second)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -450,12 +463,13 @@ func makePvcKey(ns, name string) types.NamespacedName {
// (instantiated) and thus the PV's ClaimRef cannot be completely filled-in in // (instantiated) and thus the PV's ClaimRef cannot be completely filled-in in
// this func. Therefore, the ClaimRef's name is added later in // this func. Therefore, the ClaimRef's name is added later in
// createPVCPV. // createPVCPV.
func makePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume { func MakePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume {
// Specs are expected to match this test's PersistentVolumeClaim // Specs are expected to match the test's PersistentVolumeClaim
var claimRef *v1.ObjectReference var claimRef *v1.ObjectReference
// If the reclaimPolicy is not provided, assume Retain // If the reclaimPolicy is not provided, assume Retain
if pvConfig.ReclaimPolicy == "" { if pvConfig.ReclaimPolicy == "" {
Logf("PV ReclaimPolicy unspecified, default: Retain")
pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRetain pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRetain
} }
if pvConfig.Prebind != nil { if pvConfig.Prebind != nil {
@ -467,6 +481,7 @@ func makePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume
return &v1.PersistentVolume{ return &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
GenerateName: pvConfig.NamePrefix, GenerateName: pvConfig.NamePrefix,
Labels: pvConfig.Labels,
Annotations: map[string]string{ Annotations: map[string]string{
volumehelper.VolumeGidAnnotationKey: "777", volumehelper.VolumeGidAnnotationKey: "777",
}, },
@ -491,23 +506,23 @@ func makePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume
// Note: if this PVC is intended to be pre-bound to a PV, whose name is not // Note: if this PVC is intended to be pre-bound to a PV, whose name is not
// known until the PV is instantiated, then the func CreatePVPVC will add // known until the PV is instantiated, then the func CreatePVPVC will add
// pvc.Spec.VolumeName to this claim. // pvc.Spec.VolumeName to this claim.
func MakePersistentVolumeClaim(ns string) *v1.PersistentVolumeClaim { func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.PersistentVolumeClaim {
// Specs are expected to match this test's PersistentVolume // Specs are expected to match this test's PersistentVolume
if len(cfg.AccessModes) == 0 {
Logf("AccessModes unspecified, default: all modes (RWO, RWX, ROX).")
cfg.AccessModes = append(cfg.AccessModes, v1.ReadWriteOnce, v1.ReadOnlyMany, v1.ReadOnlyMany)
}
return &v1.PersistentVolumeClaim{ return &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-", GenerateName: "pvc-",
Namespace: ns, Namespace: ns,
Annotations: map[string]string{ Annotations: cfg.Annotations,
"volume.beta.kubernetes.io/storage-class": "",
},
}, },
Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{ Selector: cfg.Selector,
v1.ReadWriteOnce, AccessModes: cfg.AccessModes,
v1.ReadOnlyMany,
v1.ReadWriteMany,
},
Resources: v1.ResourceRequirements{ Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{ Requests: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"), v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"),

19
test/e2e/persistent_volumes-disruptive.go
View File

@ -27,6 +27,7 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
@ -55,8 +56,11 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
ns string ns string
nfsServerPod *v1.Pod nfsServerPod *v1.Pod
nfsPVconfig framework.PersistentVolumeConfig nfsPVconfig framework.PersistentVolumeConfig
pvcConfig framework.PersistentVolumeClaimConfig
nfsServerIP, clientNodeIP string nfsServerIP, clientNodeIP string
clientNode *v1.Node clientNode *v1.Node
volLabel labels.Set
selector *metav1.LabelSelector
) )
BeforeEach(func() { BeforeEach(func() {
@ -64,6 +68,8 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
framework.SkipUnlessNodeCountIsAtLeast(MinNodes) framework.SkipUnlessNodeCountIsAtLeast(MinNodes)
c = f.ClientSet c = f.ClientSet
ns = f.Namespace.Name ns = f.Namespace.Name
volLabel = labels.Set{framework.VolumeSelectorKey: ns}
selector = metav1.SetAsLabelSelector(volLabel)
// Start the NFS server pod. // Start the NFS server pod.
framework.Logf("[BeforeEach] Creating NFS Server Pod") framework.Logf("[BeforeEach] Creating NFS Server Pod")
@ -74,6 +80,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
Expect(nfsServerIP).NotTo(BeEmpty()) Expect(nfsServerIP).NotTo(BeEmpty())
nfsPVconfig = framework.PersistentVolumeConfig{ nfsPVconfig = framework.PersistentVolumeConfig{
NamePrefix: "nfs-", NamePrefix: "nfs-",
Labels: volLabel,
PVSource: v1.PersistentVolumeSource{ PVSource: v1.PersistentVolumeSource{
NFS: &v1.NFSVolumeSource{ NFS: &v1.NFSVolumeSource{
Server: nfsServerIP, Server: nfsServerIP,
@ -82,6 +89,12 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
}, },
}, },
} }
pvcConfig = framework.PersistentVolumeClaimConfig{
Annotations: map[string]string{
v1.BetaStorageClassAnnotation: "",
},
Selector: selector,
}
// Get the first ready node IP that is not hosting the NFS pod. // Get the first ready node IP that is not hosting the NFS pod.
if clientNodeIP == "" { if clientNodeIP == "" {
framework.Logf("Designating test node") framework.Logf("Designating test node")
@ -111,7 +124,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
BeforeEach(func() { BeforeEach(func() {
framework.Logf("Initializing test spec") framework.Logf("Initializing test spec")
clientPod, pv, pvc = initTestCase(f, c, nfsPVconfig, ns, clientNode.Name) clientPod, pv, pvc = initTestCase(f, c, nfsPVconfig, pvcConfig, ns, clientNode.Name)
}) })
AfterEach(func() { AfterEach(func() {
@ -187,8 +200,8 @@ func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framew
} }
// initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed by the test // initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed by the test
func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) { func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
pv, pvc := framework.CreatePVPVC(c, pvConfig, ns, false) pv, pvc := framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
pod := framework.MakePod(ns, pvc.Name, true, "") pod := framework.MakePod(ns, pvc.Name, true, "")
pod.Spec.NodeName = nodeName pod.Spec.NodeName = nodeName
framework.Logf("Creating nfs client Pod %s on node %s", pod.Name, nodeName) framework.Logf("Creating nfs client Pod %s on node %s", pod.Name, nodeName)

22
test/e2e/persistent_volumes-gce.go
View File

@ -21,6 +21,8 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
@ -37,9 +39,9 @@ func verifyGCEDiskAttached(diskName string, nodeName types.NodeName) bool {
} }
// initializeGCETestSpec creates a PV, PVC, and ClientPod that will run until killed by test or clean up. // initializeGCETestSpec creates a PV, PVC, and ClientPod that will run until killed by test or clean up.
func initializeGCETestSpec(c clientset.Interface, ns string, pvConfig framework.PersistentVolumeConfig, isPrebound bool) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) { func initializeGCETestSpec(c clientset.Interface, ns string, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, isPrebound bool) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
By("Creating the PV and PVC") By("Creating the PV and PVC")
pv, pvc := framework.CreatePVPVC(c, pvConfig, ns, isPrebound) pv, pvc := framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, isPrebound)
framework.WaitOnPVandPVC(c, ns, pv, pvc) framework.WaitOnPVandPVC(c, ns, pv, pvc)
By("Creating the Client Pod") By("Creating the Client Pod")
@ -58,6 +60,9 @@ var _ = framework.KubeDescribe("PersistentVolumes:GCEPD [Volume]", func() {
pvc *v1.PersistentVolumeClaim pvc *v1.PersistentVolumeClaim
clientPod *v1.Pod clientPod *v1.Pod
pvConfig framework.PersistentVolumeConfig pvConfig framework.PersistentVolumeConfig
pvcConfig framework.PersistentVolumeClaimConfig
volLabel labels.Set
selector *metav1.LabelSelector
node types.NodeName node types.NodeName
) )
@ -66,13 +71,17 @@ var _ = framework.KubeDescribe("PersistentVolumes:GCEPD [Volume]", func() {
c = f.ClientSet c = f.ClientSet
ns = f.Namespace.Name ns = f.Namespace.Name
// Enforce binding only within test space via selector labels
volLabel = labels.Set{framework.VolumeSelectorKey: ns}
selector = metav1.SetAsLabelSelector(volLabel)
framework.SkipUnlessProviderIs("gce", "gke") framework.SkipUnlessProviderIs("gce", "gke")
By("Initializing Test Spec") By("Initializing Test Spec")
if diskName == "" {
diskName, err = framework.CreatePDWithRetry() diskName, err = framework.CreatePDWithRetry()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
pvConfig = framework.PersistentVolumeConfig{ pvConfig = framework.PersistentVolumeConfig{
NamePrefix: "gce-", NamePrefix: "gce-",
Labels: volLabel,
PVSource: v1.PersistentVolumeSource{ PVSource: v1.PersistentVolumeSource{
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
PDName: diskName, PDName: diskName,
@ -82,8 +91,13 @@ var _ = framework.KubeDescribe("PersistentVolumes:GCEPD [Volume]", func() {
}, },
Prebind: nil, Prebind: nil,
} }
pvcConfig = framework.PersistentVolumeClaimConfig{
Annotations: map[string]string{
v1.BetaStorageClassAnnotation: "",
},
Selector: selector,
} }
clientPod, pv, pvc = initializeGCETestSpec(c, ns, pvConfig, false) clientPod, pv, pvc = initializeGCETestSpec(c, ns, pvConfig, pvcConfig, false)
node = types.NodeName(clientPod.Spec.NodeName) node = types.NodeName(clientPod.Spec.NodeName)
}) })

16
test/e2e/persistent_volumes-vsphere.go
View File

@ -27,6 +27,7 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/labels"
) )
// Testing configurations of single a PV/PVC pair attached to a vSphere Disk // Testing configurations of single a PV/PVC pair attached to a vSphere Disk
@ -39,9 +40,12 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
pvc *v1.PersistentVolumeClaim pvc *v1.PersistentVolumeClaim
clientPod *v1.Pod clientPod *v1.Pod
pvConfig framework.PersistentVolumeConfig pvConfig framework.PersistentVolumeConfig
pvcConfig framework.PersistentVolumeClaimConfig
vsp *vsphere.VSphere vsp *vsphere.VSphere
err error err error
node types.NodeName node types.NodeName
volLabel labels.Set
selector *metav1.LabelSelector
) )
f := framework.NewDefaultFramework("pv") f := framework.NewDefaultFramework("pv")
@ -62,6 +66,9 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
pvc = nil pvc = nil
pv = nil pv = nil
volLabel = labels.Set{framework.VolumeSelectorKey: ns}
selector = metav1.SetAsLabelSelector(volLabel)
if vsp == nil { if vsp == nil {
vsp, err = vsphere.GetVSphere() vsp, err = vsphere.GetVSphere()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
@ -71,6 +78,7 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
pvConfig = framework.PersistentVolumeConfig{ pvConfig = framework.PersistentVolumeConfig{
NamePrefix: "vspherepv-", NamePrefix: "vspherepv-",
Labels: volLabel,
PVSource: v1.PersistentVolumeSource{ PVSource: v1.PersistentVolumeSource{
VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
VolumePath: volumePath, VolumePath: volumePath,
@ -79,9 +87,15 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
}, },
Prebind: nil, Prebind: nil,
} }
pvcConfig = framework.PersistentVolumeClaimConfig{
Annotations: map[string]string{
v1.BetaStorageClassAnnotation: "",
},
Selector: selector,
}
} }
By("Creating the PV and PVC") By("Creating the PV and PVC")
pv, pvc = framework.CreatePVPVC(c, pvConfig, ns, false) pv, pvc = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
framework.WaitOnPVandPVC(c, ns, pv, pvc) framework.WaitOnPVandPVC(c, ns, pv, pvc)
By("Creating the Client Pod") By("Creating the Client Pod")

54
test/e2e/persistent_volumes.go
View File

@ -23,6 +23,7 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
@ -85,16 +86,27 @@ func initNFSserverPod(c clientset.Interface, ns string) *v1.Pod {
}) })
} }
var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() { var _ = framework.KubeDescribe("PersistentVolumes [Volume]", func() {
// global vars for the Context()s and It()'s below // global vars for the Context()s and It()'s below
f := framework.NewDefaultFramework("pv") f := framework.NewDefaultFramework("pv")
var c clientset.Interface var (
var ns string c clientset.Interface
ns string
pvConfig framework.PersistentVolumeConfig
pvcConfig framework.PersistentVolumeClaimConfig
volLabel labels.Set
selector *metav1.LabelSelector
pv *v1.PersistentVolume
pvc *v1.PersistentVolumeClaim
)
BeforeEach(func() { BeforeEach(func() {
c = f.ClientSet c = f.ClientSet
ns = f.Namespace.Name ns = f.Namespace.Name
// Enforce binding only within test space via selector labels
volLabel = labels.Set{framework.VolumeSelectorKey: ns}
selector = metav1.SetAsLabelSelector(volLabel)
}) })
// Testing configurations of a single a PV/PVC pair, multiple evenly paired PVs/PVCs, // Testing configurations of a single a PV/PVC pair, multiple evenly paired PVs/PVCs,
@ -104,7 +116,6 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
var ( var (
nfsServerPod *v1.Pod nfsServerPod *v1.Pod
serverIP string serverIP string
pvConfig framework.PersistentVolumeConfig
) )
BeforeEach(func() { BeforeEach(func() {
@ -114,6 +125,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
framework.Logf("[BeforeEach] Configuring PersistentVolume") framework.Logf("[BeforeEach] Configuring PersistentVolume")
pvConfig = framework.PersistentVolumeConfig{ pvConfig = framework.PersistentVolumeConfig{
NamePrefix: "nfs-", NamePrefix: "nfs-",
Labels: volLabel,
PVSource: v1.PersistentVolumeSource{ PVSource: v1.PersistentVolumeSource{
NFS: &v1.NFSVolumeSource{ NFS: &v1.NFSVolumeSource{
Server: serverIP, Server: serverIP,
@ -122,17 +134,22 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
}, },
}, },
} }
pvcConfig = framework.PersistentVolumeClaimConfig{
Annotations: map[string]string{
v1.BetaStorageClassAnnotation: "",
},
Selector: selector,
}
}) })
AfterEach(func() { AfterEach(func() {
framework.DeletePodWithWait(f, c, nfsServerPod) framework.DeletePodWithWait(f, c, nfsServerPod)
pv, pvc = nil, nil
pvConfig, pvcConfig = framework.PersistentVolumeConfig{}, framework.PersistentVolumeClaimConfig{}
}) })
Context("with Single PV - PVC pairs", func() { Context("with Single PV - PVC pairs", func() {
var pv *v1.PersistentVolume
var pvc *v1.PersistentVolumeClaim
// Note: this is the only code where the pv is deleted. // Note: this is the only code where the pv is deleted.
AfterEach(func() { AfterEach(func() {
framework.Logf("AfterEach: Cleaning up test resources.") framework.Logf("AfterEach: Cleaning up test resources.")
@ -145,7 +162,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// contains the claim. Verify that the PV and PVC bind correctly, and // contains the claim. Verify that the PV and PVC bind correctly, and
// that the pod can write to the nfs volume. // that the pod can write to the nfs volume.
It("should create a non-pre-bound PV and PVC: test write access ", func() { It("should create a non-pre-bound PV and PVC: test write access ", func() {
pv, pvc = framework.CreatePVPVC(c, pvConfig, ns, false) pv, pvc = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
completeTest(f, c, ns, pv, pvc) completeTest(f, c, ns, pv, pvc)
}) })
@ -153,7 +170,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// pod that contains the claim. Verify that the PV and PVC bind // pod that contains the claim. Verify that the PV and PVC bind
// correctly, and that the pod can write to the nfs volume. // correctly, and that the pod can write to the nfs volume.
It("create a PVC and non-pre-bound PV: test write access", func() { It("create a PVC and non-pre-bound PV: test write access", func() {
pv, pvc = framework.CreatePVCPV(c, pvConfig, ns, false) pv, pvc = framework.CreatePVCPV(c, pvConfig, pvcConfig, ns, false)
completeTest(f, c, ns, pv, pvc) completeTest(f, c, ns, pv, pvc)
}) })
@ -161,7 +178,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// and a pod that contains the claim. Verify that the PV and PVC bind // and a pod that contains the claim. Verify that the PV and PVC bind
// correctly, and that the pod can write to the nfs volume. // correctly, and that the pod can write to the nfs volume.
It("create a PVC and a pre-bound PV: test write access", func() { It("create a PVC and a pre-bound PV: test write access", func() {
pv, pvc = framework.CreatePVCPV(c, pvConfig, ns, true) pv, pvc = framework.CreatePVCPV(c, pvConfig, pvcConfig, ns, true)
completeTest(f, c, ns, pv, pvc) completeTest(f, c, ns, pv, pvc)
}) })
@ -169,7 +186,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// and a pod that contains the claim. Verify that the PV and PVC bind // and a pod that contains the claim. Verify that the PV and PVC bind
// correctly, and that the pod can write to the nfs volume. // correctly, and that the pod can write to the nfs volume.
It("create a PV and a pre-bound PVC: test write access", func() { It("create a PV and a pre-bound PVC: test write access", func() {
pv, pvc = framework.CreatePVPVC(c, pvConfig, ns, true) pv, pvc = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true)
completeTest(f, c, ns, pv, pvc) completeTest(f, c, ns, pv, pvc)
}) })
}) })
@ -201,7 +218,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// Note: PVs are created before claims and no pre-binding // Note: PVs are created before claims and no pre-binding
It("should create 2 PVs and 4 PVCs: test write access", func() { It("should create 2 PVs and 4 PVCs: test write access", func() {
numPVs, numPVCs := 2, 4 numPVs, numPVCs := 2, 4
pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig) pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
framework.WaitAndVerifyBinds(c, ns, pvols, claims, true) framework.WaitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased) completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
}) })
@ -210,7 +227,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// Note: PVs are created before claims and no pre-binding // Note: PVs are created before claims and no pre-binding
It("should create 3 PVs and 3 PVCs: test write access", func() { It("should create 3 PVs and 3 PVCs: test write access", func() {
numPVs, numPVCs := 3, 3 numPVs, numPVCs := 3, 3
pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig) pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
framework.WaitAndVerifyBinds(c, ns, pvols, claims, true) framework.WaitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased) completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
}) })
@ -219,7 +236,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// Note: PVs are created before claims and no pre-binding. // Note: PVs are created before claims and no pre-binding.
It("should create 4 PVs and 2 PVCs: test write access", func() { It("should create 4 PVs and 2 PVCs: test write access", func() {
numPVs, numPVCs := 4, 2 numPVs, numPVCs := 4, 2
pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig) pvols, claims = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
framework.WaitAndVerifyBinds(c, ns, pvols, claims, true) framework.WaitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased) completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
}) })
@ -229,12 +246,9 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// Recycler, this entire context can be removed without affecting the test suite or leaving behind // Recycler, this entire context can be removed without affecting the test suite or leaving behind
// dead code. // dead code.
Context("when invoking the Recycle reclaim policy", func() { Context("when invoking the Recycle reclaim policy", func() {
var pv *v1.PersistentVolume
var pvc *v1.PersistentVolumeClaim
BeforeEach(func() { BeforeEach(func() {
pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRecycle pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRecycle
pv, pvc = framework.CreatePVPVC(c, pvConfig, ns, false) pv, pvc = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
framework.WaitOnPVandPVC(c, ns, pv, pvc) framework.WaitOnPVandPVC(c, ns, pv, pvc)
}) })
@ -246,7 +260,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
// This It() tests a scenario where a PV is written to by a Pod, recycled, then the volume checked // This It() tests a scenario where a PV is written to by a Pod, recycled, then the volume checked
// for files. If files are found, the checking Pod fails, failing the test. Otherwise, the pod // for files. If files are found, the checking Pod fails, failing the test. Otherwise, the pod
// (and test) succeed. // (and test) succeed.
It("should test that a PV becomes Available and is clean after the PVC is deleted. [Volume][Serial][Flaky]", func() { It("should test that a PV becomes Available and is clean after the PVC is deleted. [Volume] [Flaky]", func() {
By("Writing to the volume.") By("Writing to the volume.")
pod := framework.MakeWritePod(ns, pvc.Name) pod := framework.MakeWritePod(ns, pvc.Name)
pod, err := c.CoreV1().Pods(ns).Create(pod) pod, err := c.CoreV1().Pods(ns).Create(pod)
@ -257,7 +271,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
framework.DeletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeAvailable) framework.DeletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeAvailable)
By("Re-mounting the volume.") By("Re-mounting the volume.")
pvc = framework.MakePersistentVolumeClaim(ns) pvc = framework.MakePersistentVolumeClaim(pvcConfig, ns)
pvc = framework.CreatePVC(c, ns, pvc) pvc = framework.CreatePVC(c, ns, pvc)
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, 60*time.Second) err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, 60*time.Second)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())

7
test/e2e/upgrades/persistent_volumes.go
View File

@ -69,9 +69,14 @@ func (t *PersistentVolumeUpgradeTest) Setup(f *framework.Framework) {
PVSource: *t.pvSource, PVSource: *t.pvSource,
Prebind: nil, Prebind: nil,
} }
pvcConfig := framework.PersistentVolumeClaimConfig{
Annotations: map[string]string{
v1.BetaStorageClassAnnotation: "",
},
}
By("Creating the PV and PVC") By("Creating the PV and PVC")
t.pv, t.pvc = framework.CreatePVPVC(f.ClientSet, pvConfig, ns, true) t.pv, t.pvc = framework.CreatePVPVC(f.ClientSet, pvConfig, pvcConfig, ns, true)
framework.WaitOnPVandPVC(f.ClientSet, ns, t.pv, t.pvc) framework.WaitOnPVandPVC(f.ClientSet, ns, t.pv, t.pvc)
By("Consuming the PV before upgrade") By("Consuming the PV before upgrade")