From afbc8a5056642ac9fab0522743a599b7a2857e36 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Sat, 2 Mar 2019 10:53:44 -0800 Subject: [PATCH] e2e test updates for beta --- test/e2e/storage/csi_mock_volume.go | 88 +++++++++---------- test/e2e/storage/csi_volumes.go | 8 +- test/e2e/storage/drivers/csi.go | 41 +++------ test/e2e/storage/utils/deployment.go | 11 +-- .../cluster-driver-registrar/rbac.yaml | 2 +- .../external-provisioner/rbac.yaml | 4 +- .../storage-csi/gce-pd/controller_ss.yaml | 3 +- .../gce-pd/controller_ss_alpha.yaml | 54 ------------ .../csi-mock-cluster-driver-registrar.yaml | 3 +- 9 files changed, 72 insertions(+), 142 deletions(-) delete mode 100644 test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 6059723ca73..16993fa1510 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/drivers" @@ -56,7 +57,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { disableAttach bool attachLimit int registerDriver bool - podInfoVersion *string + podInfo *bool scName string nodeSelectorKey string } @@ -87,7 +88,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { cs := f.ClientSet var err error - m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfoVersion, tp.attachLimit) + m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfo, tp.attachLimit) config, testCleanup := m.driver.PrepareTest(f) m.testCleanups = append(m.testCleanups, testCleanup) m.config = config @@ -186,31 +187,31 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { } // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. - Context("CSI attach test using mock driver [Feature:CSIDriverRegistry]", func() { + Context("CSI attach test using mock driver", func() { tests := []struct { - name string - disableAttach bool - deployDriverCRD bool + name string + disableAttach bool + deployClusterRegistrar bool }{ { - name: "should not require VolumeAttach for drivers without attachment", - disableAttach: true, - deployDriverCRD: true, + name: "should not require VolumeAttach for drivers without attachment", + disableAttach: true, + deployClusterRegistrar: true, }, { - name: "should require VolumeAttach for drivers with attachment", - deployDriverCRD: true, + name: "should require VolumeAttach for drivers with attachment", + deployClusterRegistrar: true, }, { - name: "should preserve attachment policy when no CSIDriver present", - deployDriverCRD: false, + name: "should preserve attachment policy when no CSIDriver present", + deployClusterRegistrar: false, }, } for _, t := range tests { test := t It(t.name, func() { var err error - init(testParameters{registerDriver: test.deployDriverCRD, disableAttach: test.disableAttach}) + init(testParameters{registerDriver: test.deployClusterRegistrar, disableAttach: test.disableAttach}) defer cleanup() _, claim, pod := createPod() @@ -242,56 +243,49 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { } }) - Context("CSI workload information using mock driver [Feature:CSIDriverRegistry]", func() { + Context("CSI workload information using mock driver", func() { var ( - err error - podInfoV1 = "v1" - podInfoUnknown = "unknown" - podInfoEmpty = "" + err error + podInfoTrue = true + podInfoFalse = false ) tests := []struct { - name string - podInfoOnMountVersion *string - deployDriverCRD bool - expectPodInfo bool + name string + podInfoOnMount *bool + deployClusterRegistrar bool + expectPodInfo bool }{ { - name: "should not be passed when podInfoOnMountVersion=nil", - podInfoOnMountVersion: nil, - deployDriverCRD: true, - expectPodInfo: false, + name: "should not be passed when podInfoOnMount=nil", + podInfoOnMount: nil, + deployClusterRegistrar: true, + expectPodInfo: false, }, { - name: "should be passed when podInfoOnMountVersion=v1", - podInfoOnMountVersion: &podInfoV1, - deployDriverCRD: true, - expectPodInfo: true, + name: "should be passed when podInfoOnMount=true", + podInfoOnMount: &podInfoTrue, + deployClusterRegistrar: true, + expectPodInfo: true, }, { - name: "should not be passed when podInfoOnMountVersion=", - podInfoOnMountVersion: &podInfoEmpty, - deployDriverCRD: true, - expectPodInfo: false, + name: "should not be passed when podInfoOnMount=false", + podInfoOnMount: &podInfoFalse, + deployClusterRegistrar: true, + expectPodInfo: false, }, { - name: "should not be passed when podInfoOnMountVersion=", - podInfoOnMountVersion: &podInfoUnknown, - deployDriverCRD: true, - expectPodInfo: false, - }, - { - name: "should not be passed when CSIDriver does not exist", - deployDriverCRD: false, - expectPodInfo: false, + name: "should not be passed when CSIDriver does not exist", + deployClusterRegistrar: false, + expectPodInfo: false, }, } for _, t := range tests { test := t It(t.name, func() { init(testParameters{ - registerDriver: test.deployDriverCRD, + registerDriver: test.deployClusterRegistrar, scName: "csi-mock-sc-" + f.UniqueName, - podInfoVersion: test.podInfoOnMountVersion}) + podInfo: test.podInfoOnMount}) defer cleanup() diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index d4234434cf5..4581a260d0c 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -17,7 +17,7 @@ limitations under the License. package storage import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -33,7 +33,7 @@ import ( // List of testDrivers to be executed in below loop var csiTestDrivers = []func() testsuites.TestDriver{ drivers.InitHostPathCSIDriver, - func() testsuites.TestDriver { return drivers.InitGcePDCSIDriver(false /* topology enabled */) }, + drivers.InitGcePDCSIDriver, drivers.InitGcePDExternalCSIDriver, drivers.InitHostPathV0CSIDriver, // Don't run tests with mock driver (drivers.InitMockCSIDriver), it does not provide persistent storage. @@ -59,9 +59,9 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) } - Context("CSI Topology test using GCE PD driver [Feature:CSINodeInfo]", func() { + Context("CSI Topology test using GCE PD driver", func() { f := framework.NewDefaultFramework("csitopology") - driver := drivers.InitGcePDCSIDriver(true /* topology enabled */).(testsuites.DynamicPVTestDriver) // TODO (#71289) eliminate by moving this test to common test suite. + driver := drivers.InitGcePDCSIDriver().(testsuites.DynamicPVTestDriver) // TODO (#71289) eliminate by moving this test to common test suite. var ( config *testsuites.PerTestConfig testCleanup func() diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 7eb310cf027..119818f5094 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -168,18 +168,18 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per // mockCSI type mockCSIDriver struct { - driverInfo testsuites.DriverInfo - manifests []string - podInfoVersion *string - attachable bool - attachLimit int + driverInfo testsuites.DriverInfo + manifests []string + podInfo *bool + attachable bool + attachLimit int } var _ testsuites.TestDriver = &mockCSIDriver{} var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface -func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfoVersion *string, attachLimit int) testsuites.TestDriver { +func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfo *bool, attachLimit int) testsuites.TestDriver { driverManifests := []string{ "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", @@ -212,10 +212,10 @@ func InitMockCSIDriver(registerDriver, driverAttachable bool, podInfoVersion *st testsuites.CapExec: false, }, }, - manifests: driverManifests, - podInfoVersion: podInfoVersion, - attachable: driverAttachable, - attachLimit: attachLimit, + manifests: driverManifests, + podInfo: podInfo, + attachable: driverAttachable, + attachLimit: attachLimit, } } @@ -273,7 +273,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest ProvisionerContainerName: "csi-provisioner", ClusterRegistrarContainerName: "csi-cluster-driver-registrar", NodeName: config.ClientNodeName, - PodInfoVersion: m.podInfoVersion, + PodInfo: m.podInfo, } cleanup, err := f.CreateFromManifests(func(item interface{}) error { return utils.PatchCSIDeployment(f, o, item) @@ -306,17 +306,15 @@ func InitHostPathV0CSIDriver() testsuites.TestDriver { // gce-pd type gcePDCSIDriver struct { - topologyEnabled bool - driverInfo testsuites.DriverInfo + driverInfo testsuites.DriverInfo } var _ testsuites.TestDriver = &gcePDCSIDriver{} var _ testsuites.DynamicPVTestDriver = &gcePDCSIDriver{} // InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface -func InitGcePDCSIDriver(topologyEnabled bool) testsuites.TestDriver { +func InitGcePDCSIDriver() testsuites.TestDriver { return &gcePDCSIDriver{ - topologyEnabled: topologyEnabled, driverInfo: testsuites.DriverInfo{ Name: GCEPDCSIProvisionerName, FeatureTag: "[Serial]", @@ -367,13 +365,6 @@ func (g *gcePDCSIDriver) GetClaimSize() string { } func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) { - if !g.topologyEnabled { - // Topology is disabled in external-provisioner, so in a multizone cluster, a pod could be - // scheduled in a different zone from the provisioned volume, causing basic provisioning - // tests to fail. - framework.SkipIfMultizone(f.ClientSet) - } - By("deploying csi gce-pd driver") cancelLogging := testsuites.StartPodLogs(f) // It would be safer to rename the gcePD driver, but that @@ -396,13 +387,9 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", "test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", + "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", } - if g.topologyEnabled { - manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml") - } else { - manifests = append(manifests, "test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml") - } cleanup, err := f.CreateFromManifests(nil, manifests...) if err != nil { framework.Failf("deploying csi gce-pd driver: %v", err) diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index e899f2ca888..c14ba2f7976 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -18,10 +18,11 @@ package utils import ( "path" + "strconv" "strings" appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/kubernetes/test/e2e/framework" ) @@ -98,8 +99,8 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // as the snapshotter here. container.Args = append(container.Args, "--snapshotter="+o.NewDriverName) case o.ClusterRegistrarContainerName: - if o.PodInfoVersion != nil { - container.Args = append(container.Args, "--pod-info-mount-version="+*o.PodInfoVersion) + if o.PodInfo != nil { + container.Args = append(container.Args, "--pod-info-mount="+strconv.FormatBool(*o.PodInfo)) } } } @@ -163,6 +164,6 @@ type PatchCSIOptions struct { // If non-empty, all pods are forced to run on this node. NodeName string // If not nil, the argument to pass to the cluster-driver-registrar's - // pod-info-mount-version argument. - PodInfoVersion *string + // pod-info-mount argument. + PodInfo *bool } diff --git a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml index c3c593537e9..0dd03f7f96d 100644 --- a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml @@ -18,7 +18,7 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cluster-driver-registrar-runner rules: - - apiGroups: ["csi.storage.k8s.io"] + - apiGroups: ["storage.k8s.io"] resources: ["csidrivers"] verbs: ["create", "delete"] diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml index e727ee2902e..249ecc99ef9 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml @@ -42,8 +42,8 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml index 2201bfc9567..10125c77a86 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml @@ -16,7 +16,8 @@ spec: serviceAccountName: csi-controller-sa containers: - name: csi-provisioner - image: gcr.io/gke-release/csi-provisioner:v1.0.1-gke.0 + # TODO: replace with official image + image: quay.io/k8scsi/csi-provisioner:tmp-test-1.14 args: - "--v=5" - "--csi-address=/csi/csi.sock" diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml deleted file mode 100644 index 412674f534d..00000000000 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss_alpha.yaml +++ /dev/null @@ -1,54 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-gce-pd-controller -spec: - serviceName: "csi-gce-pd" - replicas: 1 - selector: - matchLabels: - app: gcp-compute-persistent-disk-csi-driver - template: - metadata: - labels: - app: gcp-compute-persistent-disk-csi-driver - spec: - serviceAccountName: csi-controller-sa - containers: - - name: csi-provisioner - image: gcr.io/gke-release/csi-provisioner:v1.0.1-gke.0 - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - - "--feature-gates=Topology=true" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-attacher - image: gcr.io/gke-release/csi-attacher:v1.0.1-gke.0 - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: gce-pd-driver - image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.3.1-gke.0 - args: - - "--v=5" - - "--endpoint=unix:/csi/csi.sock" - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/cloud-sa/cloud-sa.json" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: cloud-sa-volume - readOnly: true - mountPath: "/etc/cloud-sa" - volumes: - - name: socket-dir - emptyDir: {} - - name: cloud-sa-volume - secret: - secretName: cloud-sa diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml index 72f0ebf1bcb..75b0f31f163 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml @@ -15,7 +15,8 @@ spec: serviceAccountName: csi-mock containers: - name: csi-cluster-driver-registrar - image: quay.io/k8scsi/csi-cluster-driver-registrar:canary + # TODO: replace with official image + image: quay.io/k8scsi/csi-cluster-driver-registrar:tmp-test-1.14 args: - --v=5 - --csi-address=$(ADDRESS)