github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 14:07:14 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #68025 from mkimuram/issue/68024

Browse Source 
Make csi drivers and in-tree drivers share e2e tests
This commit is contained in:
k8s-ci-robot 2018-11-07 12:52:47 -08:00 committed by GitHub
commit 6f305d925f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 387 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/e2e/storage/BUILD
View File

@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library( go_library(
name = "go_default_library", name = "go_default_library",
srcs = [ srcs = [
"csi_objects.go",
"csi_volumes.go", "csi_volumes.go",
"empty_dir_wrapper.go", "empty_dir_wrapper.go",
"ephemeral_volume.go", "ephemeral_volume.go",
@ -73,6 +72,7 @@ go_library(
"//test/e2e/framework/providers/gce:go_default_library", "//test/e2e/framework/providers/gce:go_default_library",
"//test/e2e/framework/testfiles:go_default_library", "//test/e2e/framework/testfiles:go_default_library",
"//test/e2e/storage/drivers:go_default_library", "//test/e2e/storage/drivers:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library",
"//test/e2e/storage/testsuites:go_default_library", "//test/e2e/storage/testsuites:go_default_library",
"//test/e2e/storage/utils:go_default_library", "//test/e2e/storage/utils:go_default_library",
"//test/utils/image:go_default_library", "//test/utils/image:go_default_library",

293
test/e2e/storage/csi_volumes.go
View File

@ -19,9 +19,7 @@ package storage
import ( import (
"context" "context"
"fmt" "fmt"
"math/rand"
"regexp" "regexp"
"time"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1" storagev1 "k8s.io/api/storage/v1"
@ -32,6 +30,8 @@ import (
csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/podlogs" "k8s.io/kubernetes/test/e2e/framework/podlogs"
"k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image" imageutils "k8s.io/kubernetes/test/utils/image"
@ -42,39 +42,56 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
) )
type csiTestDriver interface { // List of testDrivers to be executed in below loop
createCSIDriver() var csiTestDrivers = []func() drivers.TestDriver{
cleanupCSIDriver() drivers.InitHostPathCSIDriver,
createStorageClassTest() testsuites.StorageClassTest drivers.InitGcePDCSIDriver,
drivers.InitGcePDExternalCSIDriver,
} }
var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{ // List of testSuites to be executed in below loop
"hostPath": initCSIHostpath, var csiTestSuites = []func() testsuites.TestSuite{
"gcePD": initCSIgcePD, testsuites.InitVolumesTestSuite,
// TODO(#70258): this is temporary until we can figure out how to make e2e tests a library testsuites.InitVolumeIOTestSuite,
"[Feature: gcePD-external]": initCSIgcePDExternal, testsuites.InitVolumeModeTestSuite,
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
} }
func csiTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
tunedPatterns := []testpatterns.TestPattern{}
for _, pattern := range patterns {
// Skip inline volume and pre-provsioned PV tests for csi drivers
if pattern.VolType == testpatterns.InlineVolume || pattern.VolType == testpatterns.PreprovisionedPV {
continue
}
tunedPatterns = append(tunedPatterns, pattern)
}
return tunedPatterns
}
// This executes testSuites for csi volumes.
var _ = utils.SIGDescribe("CSI Volumes", func() { var _ = utils.SIGDescribe("CSI Volumes", func() {
f := framework.NewDefaultFramework("csi-volumes") f := framework.NewDefaultFramework("csi-volumes")
var ( var (
cancel context.CancelFunc cancel context.CancelFunc
cs clientset.Interface cs clientset.Interface
csics csiclient.Interface
ns *v1.Namespace ns *v1.Namespace
node v1.Node
config framework.VolumeTestConfig config framework.VolumeTestConfig
) )
BeforeEach(func() { BeforeEach(func() {
ctx, c := context.WithCancel(context.Background()) ctx, c := context.WithCancel(context.Background())
cancel = c cancel = c
cs = f.ClientSet cs = f.ClientSet
csics = f.CSIClientSet
ns = f.Namespace ns = f.Namespace
config = framework.VolumeTestConfig{
Namespace: ns.Name,
Prefix: "csi",
}
// Debugging of the following tests heavily depends on the log output // Debugging of the following tests heavily depends on the log output
// of the different containers. Therefore include all of that in log // of the different containers. Therefore include all of that in log
// files (when using --report-dir, as in the CI) or the output stream // files (when using --report-dir, as in the CI) or the output stream
@ -100,69 +117,50 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
if framework.TestContext.ReportDir == "" { if framework.TestContext.ReportDir == "" {
podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter) podlogs.WatchPods(ctx, cs, ns.Name, GinkgoWriter)
} }
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
node = nodes.Items[rand.Intn(len(nodes.Items))]
config = framework.VolumeTestConfig{
Namespace: ns.Name,
Prefix: "csi",
// TODO(#70259): this needs to be parameterized so only hostpath sets node name
ClientNodeName: node.Name,
ServerNodeName: node.Name,
WaitForCompletion: true,
}
}) })
AfterEach(func() { AfterEach(func() {
cancel() cancel()
}) })
for driverName, initCSIDriver := range csiTestDrivers { for _, initDriver := range csiTestDrivers {
curDriverName := driverName curDriver := initDriver()
curInitCSIDriver := initCSIDriver Context(fmt.Sprintf(drivers.GetDriverNameWithFeatureTags(curDriver)), func() {
driver := curDriver
Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() {
var (
driver csiTestDriver
)
BeforeEach(func() { BeforeEach(func() {
driver = curInitCSIDriver(f, config) // setupDriver
driver.createCSIDriver() drivers.SetCommonDriverParameters(driver, f, config)
driver.CreateDriver()
}) })
AfterEach(func() { AfterEach(func() {
driver.cleanupCSIDriver() // Cleanup driver
driver.CleanupDriver()
}) })
It("should provision storage", func() { testsuites.RunTestSuite(f, config, driver, csiTestSuites, csiTunePattern)
t := driver.createStorageClassTest()
claim := newClaim(t, ns.GetName(), "")
var class *storagev1.StorageClass
if t.StorageClassName == "" {
class = newStorageClass(t, ns.GetName(), "")
claim.Spec.StorageClassName = &class.ObjectMeta.Name
} else {
scName := t.StorageClassName
claim.Spec.StorageClassName = &scName
}
testsuites.TestDynamicProvisioning(t, cs, claim, class)
})
}) })
} }
// The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12.
Context("CSI attach test using HostPath driver [Feature:CSISkipAttach]", func() { Context("CSI attach test using HostPath driver [Feature:CSISkipAttach]", func() {
var ( var (
driver csiTestDriver cs clientset.Interface
csics csiclient.Interface
driver drivers.TestDriver
) )
BeforeEach(func() { BeforeEach(func() {
driver = initCSIHostpath(f, config) cs = f.ClientSet
driver.createCSIDriver() csics = f.CSIClientSet
driver = drivers.InitHostPathCSIDriver()
drivers.SetCommonDriverParameters(driver, f, config)
driver.CreateDriver()
}) })
AfterEach(func() { AfterEach(func() {
driver.cleanupCSIDriver() driver.CleanupDriver()
}) })
tests := []struct { tests := []struct {
@ -194,15 +192,27 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
test := t test := t
It(test.name, func() { It(test.name, func() {
if test.driverExists { if test.driverExists {
driver := createCSIDriver(csics, "csi-hostpath-"+f.UniqueName, test.driverAttachable) csiDriver := createCSIDriver(csics, "csi-hostpath-"+f.UniqueName, test.driverAttachable)
if driver != nil { if csiDriver != nil {
defer csics.CsiV1alpha1().CSIDrivers().Delete(driver.Name, nil) defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil)
} }
} }
By("Creating pod") By("Creating pod")
t := driver.createStorageClassTest() var sc *storagev1.StorageClass
class, claim, pod := startPausePod(cs, t, ns.Name) if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok {
sc = dDriver.GetDynamicProvisionStorageClass("")
}
nodeName := driver.GetDriverInfo().Config.ClientNodeName
scTest := testsuites.StorageClassTest{
Name: driver.GetDriverInfo().Name,
Provisioner: sc.Provisioner,
Parameters: sc.Parameters,
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
NodeName: nodeName,
}
class, claim, pod := startPausePod(cs, scTest, ns.Name)
if class != nil { if class != nil {
defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) defer cs.StorageV1().StorageClasses().Delete(class.Name, nil)
} }
@ -223,7 +233,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
By("Checking if VolumeAttachment was created for the pod") By("Checking if VolumeAttachment was created for the pod")
// Check that VolumeAttachment does not exist // Check that VolumeAttachment does not exist
handle := getVolumeHandle(cs, claim) handle := getVolumeHandle(cs, claim)
attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.Provisioner, node.Name))) attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, scTest.Provisioner, nodeName)))
attachmentName := fmt.Sprintf("csi-%x", attachmentHash) attachmentName := fmt.Sprintf("csi-%x", attachmentHash)
_, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{})
if err != nil { if err != nil {
@ -326,164 +336,3 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns str
framework.ExpectNoError(err, "Failed to create pod: %v", err) framework.ExpectNoError(err, "Failed to create pod: %v", err)
return class, claim, pod return class, claim, pod
} }
type hostpathCSIDriver struct {
f *framework.Framework
config framework.VolumeTestConfig
cleanup func()
}
func initCSIHostpath(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver {
return &hostpathCSIDriver{
f: f,
config: config,
}
}
func (h *hostpathCSIDriver) createStorageClassTest() testsuites.StorageClassTest {
return testsuites.StorageClassTest{
Name: "csi-hostpath",
Parameters: map[string]string{},
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
// The hostpath driver only works when everything runs on a single node.
NodeName: h.config.ServerNodeName,
// Provisioner and storage class name must match what's used in
// csi-storageclass.yaml, plus the test-specific suffix.
Provisioner: "csi-hostpath-" + h.f.UniqueName,
StorageClassName: "csi-hostpath-sc-" + h.f.UniqueName,
}
}
func (h *hostpathCSIDriver) createCSIDriver() {
By("deploying csi hostpath driver")
// TODO (?): the storage.csi.image.version and storage.csi.image.registry
// settings are ignored for this test. We could patch the image definitions.
o := utils.PatchCSIOptions{
OldDriverName: "csi-hostpath",
NewDriverName: "csi-hostpath-" + h.f.UniqueName,
DriverContainerName: "hostpath",
ProvisionerContainerName: "csi-provisioner",
NodeName: h.config.ServerNodeName,
}
cleanup, err := h.f.CreateFromManifests(func(item interface{}) error {
return utils.PatchCSIDeployment(h.f, o, item)
},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml",
)
h.cleanup = cleanup
if err != nil {
framework.Failf("deploying csi hostpath driver: %v", err)
}
}
func (h *hostpathCSIDriver) cleanupCSIDriver() {
if h.cleanup != nil {
By("uninstalling csi hostpath driver")
h.cleanup()
}
}
type gcePDCSIDriver struct {
f *framework.Framework
config framework.VolumeTestConfig
cleanup func()
}
func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver {
cs := f.ClientSet
framework.SkipUnlessProviderIs("gce", "gke")
framework.SkipIfMultizone(cs)
// TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys.
createGCESecrets(cs, config)
framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute)
return &gcePDCSIDriver{
f: f,
config: config,
}
}
func (g *gcePDCSIDriver) createStorageClassTest() testsuites.StorageClassTest {
return testsuites.StorageClassTest{
Name: "com.google.csi.gcepd",
// *Not* renaming the driver, see below.
Provisioner: "com.google.csi.gcepd",
Parameters: map[string]string{"type": "pd-standard"},
ClaimSize: "5Gi",
ExpectedSize: "5Gi",
}
}
func (g *gcePDCSIDriver) createCSIDriver() {
By("deploying gce-pd driver")
// It would be safer to rename the gcePD driver, but that
// hasn't been done before either and attempts to do so now led to
// errors during driver registration, therefore it is disabled
// by passing a nil function below.
//
// These are the options which would have to be used:
// o := utils.PatchCSIOptions{
// OldDriverName: "com.google.csi.gcepd",
// NewDriverName: "com.google.csi.gcepd-" + g.f.UniqueName,
// DriverContainerName: "gce-driver",
// ProvisionerContainerName: "csi-external-provisioner",
// }
cleanup, err := g.f.CreateFromManifests(nil,
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml",
)
g.cleanup = cleanup
if err != nil {
framework.Failf("deploying csi hostpath driver: %v", err)
}
}
func (g *gcePDCSIDriver) cleanupCSIDriver() {
By("uninstalling gce-pd driver")
if g.cleanup != nil {
g.cleanup()
}
}
type gcePDCSIDriverExternal struct {
}
func initCSIgcePDExternal(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver {
cs := f.ClientSet
framework.SkipUnlessProviderIs("gce", "gke")
framework.SkipIfMultizone(cs)
return &gcePDCSIDriverExternal{}
}
func (g *gcePDCSIDriverExternal) createStorageClassTest() testsuites.StorageClassTest {
return testsuites.StorageClassTest{
Name: "com.google.csi.gcepd",
Provisioner: "com.google.csi.gcepd",
Parameters: map[string]string{"type": "pd-standard"},
ClaimSize: "5Gi",
ExpectedSize: "5Gi",
}
}
func (g *gcePDCSIDriverExternal) createCSIDriver() {
}
func (g *gcePDCSIDriverExternal) cleanupCSIDriver() {
}

4
test/e2e/storage/drivers/BUILD
View File

@ -4,6 +4,8 @@ go_library(
name = "go_default_library", name = "go_default_library",
srcs = [ srcs = [
"base.go", "base.go",
"csi.go",
"csi_objects.go",
"in_tree.go", "in_tree.go",
], ],
importpath = "k8s.io/kubernetes/test/e2e/storage/drivers", importpath = "k8s.io/kubernetes/test/e2e/storage/drivers",
@ -17,7 +19,9 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library", "//test/e2e/framework:go_default_library",
"//test/e2e/storage/testpatterns:go_default_library", "//test/e2e/storage/testpatterns:go_default_library",
"//test/e2e/storage/utils:go_default_library", "//test/e2e/storage/utils:go_default_library",

281
test/e2e/storage/drivers/csi.go Normal file
View File

@ -0,0 +1,281 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
* This file defines various csi volume test drivers for TestSuites.
*
* There are two ways, how to prepare test drivers:
* 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...)
* It creates a server pod which defines one volume for the tests.
* These tests work only when privileged containers are allowed, exporting
* various filesystems (NFS, GlusterFS, ...) usually needs some mounting or
* other privileged magic in the server pod.
*
* Note that the server containers are for testing purposes only and should not
* be used in production.
*
* 2) With server or cloud provider outside of Kubernetes (Cinder, GCE, AWS, Azure, ...)
* Appropriate server or cloud provider must exist somewhere outside
* the tested Kubernetes cluster. CreateVolume will create a new volume to be
* used in the TestSuites for inlineVolume or DynamicPV tests.
*/
package drivers
import (
"fmt"
"math/rand"
"time"
. "github.com/onsi/ginkgo"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/utils"
)
// hostpathCSI
type hostpathCSIDriver struct {
cleanup func()
driverInfo DriverInfo
}
var _ TestDriver = &hostpathCSIDriver{}
var _ DynamicPVTestDriver = &hostpathCSIDriver{}
// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
func InitHostPathCSIDriver() TestDriver {
return &hostpathCSIDriver{
driverInfo: DriverInfo{
Name: "csi-hostpath",
FeatureTag: "",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
),
IsPersistent: true,
IsFsGroupSupported: false,
IsBlockSupported: false,
},
}
}
func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo {
return &h.driverInfo
}
func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
}
func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
provisioner := h.driverInfo.Name + h.driverInfo.Framework.UniqueName
parameters := map[string]string{}
ns := h.driverInfo.Framework.Namespace.Name
suffix := fmt.Sprintf("%s-sc", provisioner)
return getStorageClass(provisioner, parameters, nil, ns, suffix)
}
func (h *hostpathCSIDriver) CreateDriver() {
By("deploying csi hostpath driver")
f := h.driverInfo.Framework
cs := f.ClientSet
// pods should be scheduled on the node
nodes := framework.GetReadySchedulableNodesOrDie(cs)
node := nodes.Items[rand.Intn(len(nodes.Items))]
h.driverInfo.Config.ClientNodeName = node.Name
h.driverInfo.Config.ServerNodeName = node.Name
// TODO (?): the storage.csi.image.version and storage.csi.image.registry
// settings are ignored for this test. We could patch the image definitions.
o := utils.PatchCSIOptions{
OldDriverName: h.driverInfo.Name,
NewDriverName: h.driverInfo.Name + h.driverInfo.Framework.UniqueName,
DriverContainerName: "hostpath",
ProvisionerContainerName: "csi-provisioner",
NodeName: h.driverInfo.Config.ServerNodeName,
}
cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error {
return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item)
},
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
)
h.cleanup = cleanup
if err != nil {
framework.Failf("deploying csi hostpath driver: %v", err)
}
}
func (h *hostpathCSIDriver) CleanupDriver() {
if h.cleanup != nil {
By("uninstalling csi hostpath driver")
h.cleanup()
}
}
// gce-pd
type gcePDCSIDriver struct {
cleanup func()
driverInfo DriverInfo
}
var _ TestDriver = &gcePDCSIDriver{}
var _ DynamicPVTestDriver = &gcePDCSIDriver{}
// InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface
func InitGcePDCSIDriver() TestDriver {
return &gcePDCSIDriver{
driverInfo: DriverInfo{
Name: "com.google.csi.gcepd",
FeatureTag: "[Serial]",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext2",
"ext3",
"ext4",
"xfs",
),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: false,
},
}
}
func (g *gcePDCSIDriver) GetDriverInfo() *DriverInfo {
return &g.driverInfo
}
func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
f := g.driverInfo.Framework
cs := f.ClientSet
config := g.driverInfo.Config
framework.SkipUnlessProviderIs("gce", "gke")
framework.SkipIfMultizone(cs)
// TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys.
createGCESecrets(cs, config)
framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute)
}
func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
ns := g.driverInfo.Framework.Namespace.Name
provisioner := g.driverInfo.Name
suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
parameters := map[string]string{"type": "pd-standard"}
return getStorageClass(provisioner, parameters, nil, ns, suffix)
}
func (g *gcePDCSIDriver) CreateDriver() {
By("deploying csi gce-pd driver")
// It would be safer to rename the gcePD driver, but that
// hasn't been done before either and attempts to do so now led to
// errors during driver registration, therefore it is disabled
// by passing a nil function below.
//
// These are the options which would have to be used:
// o := utils.PatchCSIOptions{
// OldDriverName: "com.google.csi.gcepd",
// NewDriverName: "com.google.csi.gcepd-" + g.f.UniqueName,
// DriverContainerName: "gce-driver",
// ProvisionerContainerName: "csi-external-provisioner",
// }
cleanup, err := g.driverInfo.Framework.CreateFromManifests(nil,
"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml",
"test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml",
)
g.cleanup = cleanup
if err != nil {
framework.Failf("deploying csi gce-pd driver: %v", err)
}
}
func (g *gcePDCSIDriver) CleanupDriver() {
By("uninstalling gce-pd driver")
if g.cleanup != nil {
g.cleanup()
}
}
// gcePd-external
type gcePDExternalCSIDriver struct {
driverInfo DriverInfo
}
var _ TestDriver = &gcePDExternalCSIDriver{}
var _ DynamicPVTestDriver = &gcePDExternalCSIDriver{}
// InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface
func InitGcePDExternalCSIDriver() TestDriver {
return &gcePDExternalCSIDriver{
driverInfo: DriverInfo{
Name: "com.google.csi.gcepd",
// TODO(#70258): this is temporary until we can figure out how to make e2e tests a library
FeatureTag: "[Serial][Feature: gcePD-external]",
MaxFileSize: testpatterns.FileSizeMedium,
SupportedFsType: sets.NewString(
"", // Default fsType
"ext2",
"ext3",
"ext4",
"xfs",
),
IsPersistent: true,
IsFsGroupSupported: true,
IsBlockSupported: false,
},
}
}
func (g *gcePDExternalCSIDriver) GetDriverInfo() *DriverInfo {
return &g.driverInfo
}
func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
framework.SkipUnlessProviderIs("gce", "gke")
framework.SkipIfMultizone(g.driverInfo.Framework.ClientSet)
}
func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
ns := g.driverInfo.Framework.Namespace.Name
provisioner := g.driverInfo.Name
suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
parameters := map[string]string{"type": "pd-standard"}
return getStorageClass(provisioner, parameters, nil, ns, suffix)
}
func (g *gcePDExternalCSIDriver) CreateDriver() {
}
func (g *gcePDExternalCSIDriver) CleanupDriver() {
}

2
test/e2e/storage/csi_objects.go → test/e2e/storage/drivers/csi_objects.go
View File

@ -17,7 +17,7 @@ limitations under the License.
// This file is used to deploy the CSI hostPath plugin // This file is used to deploy the CSI hostPath plugin
// More Information: https://github.com/kubernetes-csi/drivers/tree/master/pkg/hostpath // More Information: https://github.com/kubernetes-csi/drivers/tree/master/pkg/hostpath
package storage package drivers
import ( import (
"flag" "flag"

7
test/e2e/storage/in_tree_volumes.go
View File

@ -23,6 +23,7 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/drivers" "k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
"k8s.io/kubernetes/test/e2e/storage/testsuites" "k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils" "k8s.io/kubernetes/test/e2e/storage/utils"
) )
@ -53,6 +54,10 @@ var testSuites = []func() testsuites.TestSuite{
testsuites.InitProvisioningTestSuite, testsuites.InitProvisioningTestSuite,
} }
func intreeTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
return patterns
}
// This executes testSuites for in-tree volumes. // This executes testSuites for in-tree volumes.
var _ = utils.SIGDescribe("In-tree Volumes", func() { var _ = utils.SIGDescribe("In-tree Volumes", func() {
f := framework.NewDefaultFramework("volumes") f := framework.NewDefaultFramework("volumes")
@ -86,7 +91,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() {
driver.CleanupDriver() driver.CleanupDriver()
}) })
testsuites.RunTestSuite(f, config, driver, testSuites) testsuites.RunTestSuite(f, config, driver, testSuites, intreeTunePattern)
}) })
} }
}) })

8
test/e2e/storage/testsuites/base.go
View File

@ -66,12 +66,12 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string {
} }
// RunTestSuite runs all testpatterns of all testSuites for a driver // RunTestSuite runs all testpatterns of all testSuites for a driver
func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite) { func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) {
for _, testSuiteInit := range tsInits { for _, testSuiteInit := range tsInits {
suite := testSuiteInit() suite := testSuiteInit()
tsInfo := suite.getTestSuiteInfo() patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns)
for _, pattern := range tsInfo.testPatterns { for _, pattern := range patterns {
suite.execTest(driver, pattern) suite.execTest(driver, pattern)
} }
} }
@ -164,7 +164,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat
case testpatterns.DynamicPV: case testpatterns.DynamicPV:
framework.Logf("Creating resource for dynamic PV") framework.Logf("Creating resource for dynamic PV")
if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok { if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok {
claimSize := "2Gi" claimSize := "5Gi"
r.sc = dDriver.GetDynamicProvisionStorageClass(fsType) r.sc = dDriver.GetDynamicProvisionStorageClass(fsType)
By("creating a StorageClass " + r.sc.Name) By("creating a StorageClass " + r.sc.Name)

2
test/e2e/storage/testsuites/provisioning.go
View File

@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt
framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name) framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
} }
p.driver = driver p.driver = driver
p.claimSize = "2Gi" p.claimSize = "5Gi"
p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name) p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name)
p.pvc.Spec.StorageClassName = &p.sc.Name p.pvc.Spec.StorageClassName = &p.sc.Name
framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc) framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)

19
test/e2e/storage/testsuites/subpath.go
View File

@ -19,6 +19,7 @@ package testsuites
import ( import (
"fmt" "fmt"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
@ -315,7 +316,8 @@ func testSubPath(input *subPathTestInput) {
}) })
It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() { It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() {
if input.volType == "hostPath" || input.volType == "hostPathSymlink" { if strings.HasPrefix(input.volType, "hostPath") || strings.HasPrefix(input.volType, "csi-hostpath") {
// TODO: This skip should be removed once #61446 is fixed
framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType) framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType)
} }
testSubpathReconstruction(input.f, input.pod, true) testSubpathReconstruction(input.f, input.pod, true)
@ -380,10 +382,23 @@ func TestBasicSubpathFile(f *framework.Framework, contents string, pod *v1.Pod,
Expect(err).NotTo(HaveOccurred(), "while deleting pod") Expect(err).NotTo(HaveOccurred(), "while deleting pod")
} }
func generateSuffixForPodName(s string) string {
// Pod name must:
// 1. consist of lower case alphanumeric characters or '-',
// 2. start and end with an alphanumeric character.
// (e.g. 'my-name', or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')
// Therefore, suffix is generated by following steps:
// 1. all strings other than [A-Za-z0-9] is replaced with "-",
// 2. add lower case alphanumeric characters at the end ('-[a-z0-9]{4}' is added),
// 3. convert the entire strings to lower case.
re := regexp.MustCompile("[^A-Za-z0-9]")
return strings.ToLower(fmt.Sprintf("%s-%s", re.ReplaceAllString(s, "-"), rand.String(4)))
}
// SubpathTestPod returns a pod spec for subpath tests // SubpathTestPod returns a pod spec for subpath tests
func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource, privilegedSecurityContext bool) *v1.Pod { func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource, privilegedSecurityContext bool) *v1.Pod {
var ( var (
suffix = strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4))) suffix = generateSuffixForPodName(volumeType)
gracePeriod = int64(1) gracePeriod = int64(1)
probeVolumeName = "liveness-probe-volume" probeVolumeName = "liveness-probe-volume"
) )

2
test/e2e/storage/testsuites/volumemode.go
View File

@ -197,7 +197,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter
} }
s.sc.VolumeBindingMode = &volBindMode s.sc.VolumeBindingMode = &volBindMode
claimSize := "2Gi" claimSize := "5Gi"
s.pvc = getClaim(claimSize, ns.Name) s.pvc = getClaim(claimSize, ns.Name)
s.pvc.Spec.StorageClassName = &s.sc.Name s.pvc.Spec.StorageClassName = &s.sc.Name
s.pvc.Spec.VolumeMode = &volMode s.pvc.Spec.VolumeMode = &volMode