From 6c6930a0880f0e963e82d701b8987b15ebb71fd0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 25 Jul 2019 12:57:06 +0200 Subject: [PATCH 1/6] e2e storage: synchronize .yaml files with latest upstream releases This ensures that the files are in sync with: hostpath: v1.2.0-rc3 external-attacher: v2.0.1 external-provisioner: v1.3.0 external-resizer: v0.2.0 external-snapshotter: v1.2.0 driver-registrar/rbac.yaml is obsolete because only node-driver-registrar is in use now and does not need RBAC rules. mock/e2e-test-rbac.yaml was not used anywhere. The README.md files were updated to indicate that these really are files copied from elsewhere. To avoid the need to constantly edit these files on each update, is used as placeholder in the URL. --- test/e2e/storage/drivers/csi.go | 9 +- .../storage-csi/driver-registrar/README.md | 1 - .../storage-csi/driver-registrar/rbac.yaml | 51 ---------- .../storage-csi/external-attacher/README.md | 2 +- .../external-provisioner/README.md | 2 +- .../storage-csi/external-resizer/README.md | 2 +- .../storage-csi/external-resizer/rbac.yaml | 5 +- .../external-snapshotter/README.md | 2 +- .../external-snapshotter/rbac.yaml | 95 ++++++++++++------- .../gce-pd/csi-controller-rbac.yaml | 3 - .../storage-csi/gce-pd/node_ds.yaml | 1 - .../hostpath-v0/csi-hostpathplugin.yaml | 1 - .../hostpath/hostpath-v0/e2e-test-rbac.yaml | 3 - .../hostpath/csi-hostpath-attacher.yaml | 17 +++- ...thplugin.yaml => csi-hostpath-plugin.yaml} | 71 +++++++++++++- .../hostpath/csi-hostpath-provisioner.yaml | 26 +++-- .../hostpath/hostpath/e2e-test-rbac.yaml | 3 - .../storage-csi/mock/e2e-test-rbac.yaml | 12 --- 18 files changed, 164 insertions(+), 142 deletions(-) delete mode 100644 test/e2e/testing-manifests/storage-csi/driver-registrar/README.md delete mode 100644 test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml rename test/e2e/testing-manifests/storage-csi/hostpath/hostpath/{csi-hostpathplugin.yaml => csi-hostpath-plugin.yaml} (51%) delete mode 100644 test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index b7f9ae7933d..87d511a45f3 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -97,7 +97,6 @@ func InitHostPathCSIDriver() testsuites.TestDriver { } return initHostPathCSIDriver("csi-hostpath", capabilities, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", @@ -106,7 +105,7 @@ func InitHostPathCSIDriver() testsuites.TestDriver { "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", ) } @@ -205,7 +204,6 @@ var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver { driverManifests := []string{ - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", @@ -329,7 +327,9 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest func InitHostPathV0CSIDriver() testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath-v0", map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true}, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + // Using the current set of rbac.yaml files is problematic here because they don't + // match the version of the rules that were written for the releases of external-attacher + // and external-provisioner that we are using here. It happens to work in practice... "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-attacher.yaml", @@ -423,7 +423,6 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes createGCESecrets(f.ClientSet, f.Namespace.Name) manifests := []string{ - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml", diff --git a/test/e2e/testing-manifests/storage-csi/driver-registrar/README.md b/test/e2e/testing-manifests/storage-csi/driver-registrar/README.md deleted file mode 100644 index f6566a99a50..00000000000 --- a/test/e2e/testing-manifests/storage-csi/driver-registrar/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is (or will be) https://github.com/kubernetes-csi/driver-registrar/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml b/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml deleted file mode 100644 index cd256039555..00000000000 --- a/test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# This YAML file contains all RBAC objects that are necessary to run external -# CSI provisioner. -# -# In production, each CSI driver deployment has to be customized: -# - to avoid conflicts, use non-default namespace and different names -# for non-namespaced entities like the ClusterRole -# - decide whether the deployment replicates the external CSI -# provisioner, in which case leadership election must be enabled; -# this influences the RBAC setup, see below - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-node-sa - # replace with non-default namespace name - namespace: default - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: driver-registrar-runner -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - # The following permissions are only needed when running - # driver-registrar without the --kubelet-registration-path - # parameter, i.e. when using driver-registrar instead of - # kubelet to update the csi.volume.kubernetes.io/nodeid - # annotation. That mode of operation is going to be deprecated - # and should not be used anymore, but is needed on older - # Kubernetes versions. - # - apiGroups: [""] - # resources: ["nodes"] - # verbs: ["get", "update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-driver-registrar-role -subjects: - - kind: ServiceAccount - name: csi-node-sa - # replace with non-default namespace name - namespace: default -roleRef: - kind: ClusterRole - name: driver-registrar-runner - apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md b/test/e2e/testing-manifests/storage-csi/external-attacher/README.md index 705533f5914..ee21fe01268 100644 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md +++ b/test/e2e/testing-manifests/storage-csi/external-attacher/README.md @@ -1 +1 @@ -The original file is (or will be) https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml +The original file is https://github.com/kubernetes-csi/external-attacher/blob//deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md b/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md index adc2091df06..d019a806141 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md @@ -1 +1 @@ -The original file is (or will be) https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml +The original file is https://github.com/kubernetes-csi/external-provisioner/blob//deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md index f7029912efb..f4407c71391 100644 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md @@ -1 +1 @@ -The original file is (or will be) https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml +The original file is https://github.com/kubernetes-csi/external-resizer/blob//deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml index fa0600a1dee..17ed01f8381 100644 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml @@ -36,9 +36,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -59,7 +56,7 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- -# Resizer must be able to work with leases in current namespace +# Resizer must be able to work with end point in current namespace # if (and only if) leadership election is enabled kind: Role apiVersion: rbac.authorization.k8s.io/v1 diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md index 90dab9588d8..691e511ef28 100644 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md @@ -1 +1 @@ -The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml +The original file is https://github.com/kubernetes-csi/external-snapshotter/blob//deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml index a8f85212931..6a07ee32141 100644 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml @@ -20,36 +20,36 @@ metadata: # rename if there are conflicts name: external-snapshotter-runner rules: -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] --- kind: ClusterRoleBinding @@ -57,12 +57,39 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-snapshotter-role subjects: -- kind: ServiceAccount - name: csi-snapshotter - # replace with non-default namespace name - namespace: default + - kind: ServiceAccount + name: csi-snapshotter + # replace with non-default namespace name + namespace: default roleRef: kind: ClusterRole # change the name also here if the ClusterRole gets renamed name: external-snapshotter-runner apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default # TODO: replace with the namespace you want for your sidecar + name: external-snapshotter-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-snapshotter-leaderelection + namespace: default # TODO: replace with the namespace you want for your sidecar +subjects: + - kind: ServiceAccount + name: csi-snapshotter + namespace: default # TODO: replace with the namespace you want for your sidecar +roleRef: + kind: Role + name: external-snapshotter-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml index 55417c4b4be..f9918530634 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml @@ -69,9 +69,6 @@ subjects: - kind: ServiceAccount name: csi-controller-sa namespace: default - - kind: ServiceAccount - name: csi-node-sa - namespace: default roleRef: kind: ClusterRole name: e2e-test-privileged-psp diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml index 8509cba5d5c..3737273583e 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml @@ -11,7 +11,6 @@ spec: labels: app: gcp-compute-persistent-disk-csi-driver spec: - serviceAccountName: csi-node-sa containers: - name: csi-driver-registrar image: gcr.io/gke-release/csi-node-driver-registrar:v1.1.0-gke.0 diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml index 2af3382adbd..820041a9284 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml @@ -11,7 +11,6 @@ spec: labels: app: csi-hostpathplugin spec: - serviceAccountName: csi-node-sa hostNetwork: true containers: - name: driver-registrar diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml index aa008ecac29..1b0ac3744a4 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml @@ -7,9 +7,6 @@ subjects: - kind: ServiceAccount name: csi-attacher namespace: default - - kind: ServiceAccount - name: csi-node-sa - namespace: default - kind: ServiceAccount name: csi-provisioner namespace: default diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml index 9462bea0c25..d7c36d2ec6c 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml @@ -27,20 +27,27 @@ spec: labels: app: csi-hostpath-attacher spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname serviceAccountName: csi-attacher containers: - name: csi-attacher image: quay.io/k8scsi/csi-attacher:v1.2.0 args: - --v=5 - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: Always + - --csi-address=/csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir + volumes: - hostPath: path: /var/lib/kubelet/plugins/csi-hostpath diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml similarity index 51% rename from test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml rename to test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 913a3b42875..ef5d3222778 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -1,8 +1,31 @@ -kind: DaemonSet +# Service defined here, plus serviceName below in StatefulSet, +# are needed only because of condition explained in +# https://github.com/kubernetes/kubernetes/issues/69608 + +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpathplugin + labels: + app: csi-hostpathplugin +spec: + selector: + app: csi-hostpathplugin + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet apiVersion: apps/v1 metadata: name: csi-hostpathplugin spec: + serviceName: "csi-hostpathplugin" + # One replica only: + # Host path driver only works when everything runs + # on a single node. We achieve that by starting it once and then + # co-locate all other pods via inter-pod affinity + replicas: 1 selector: matchLabels: app: csi-hostpathplugin @@ -11,30 +34,38 @@ spec: labels: app: csi-hostpathplugin spec: - serviceAccountName: csi-node-sa hostNetwork: true containers: - - name: driver-registrar + - name: node-driver-registrar image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"] args: - --v=5 - --csi-address=/csi/csi.sock - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock + securityContext: + privileged: true env: - name: KUBE_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - imagePullPolicy: Always volumeMounts: - mountPath: /csi name: socket-dir - mountPath: /registration name: registration-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - name: hostpath image: quay.io/k8scsi/hostpathplugin:v1.2.0-rc5 args: + - "--drivername=hostpath.csi.k8s.io" - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" @@ -46,9 +77,20 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - imagePullPolicy: Always securityContext: privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 volumeMounts: - mountPath: /csi name: socket-dir @@ -58,6 +100,19 @@ spec: - mountPath: /var/lib/kubelet/plugins mountPropagation: Bidirectional name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=9898 + volumes: - hostPath: path: /var/lib/kubelet/plugins/csi-hostpath @@ -75,3 +130,9 @@ spec: path: /var/lib/kubelet/plugins type: Directory name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-hostpath-data/ + type: DirectoryOrCreate + name: csi-data-dir diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml index 11ab11ef25a..dd9b3e3b062 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml @@ -1,12 +1,12 @@ kind: Service apiVersion: v1 metadata: - name: csi-hostpath-provisioner + name: csi-hostpath-provisioner labels: - app: csi-hostpath-provisioner + app: csi-hostpath-provisioner spec: selector: - app: csi-hostpath-provisioner + app: csi-hostpath-provisioner ports: - name: dummy port: 12345 @@ -27,19 +27,25 @@ spec: labels: app: csi-hostpath-provisioner spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname serviceAccountName: csi-provisioner containers: - name: csi-provisioner # TODO: replace with official 1.4.0 release when ready image: quay.io/k8scsi/csi-provisioner:v1.4.0-rc1 args: - - "--provisioner=csi-hostpath" - - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: Always + - -v=5 + - --csi-address=/csi/csi.sock + - --connection-timeout=15s volumeMounts: - mountPath: /csi name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml index f93b8de13cd..dde7ce78258 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml @@ -7,9 +7,6 @@ subjects: - kind: ServiceAccount name: csi-attacher namespace: default - - kind: ServiceAccount - name: csi-node-sa - namespace: default - kind: ServiceAccount name: csi-provisioner namespace: default diff --git a/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml deleted file mode 100644 index 0c4449272b8..00000000000 --- a/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: psp-csi-mock-role -subjects: - - kind: ServiceAccount - name: csi-driver-registrar - namespace: default -roleRef: - kind: ClusterRole - name: e2e-test-privileged-psp - apiGroup: rbac.authorization.k8s.io From cf125a2db35430edeebce5d10cdce2061e7511c7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 25 Jul 2019 15:03:06 +0200 Subject: [PATCH 2/6] e2e storage: enable testing of ephemeral inline volumes with hostpath CSI driver We need the 1.2.0 driver for that because that has support for detecting the volume mode dynamically, and we need to deploy a CSIDriver object which enables pod info (for the dynamic detection) and both modes (to satisfy the new mode sanity check). --- test/e2e/storage/csi_volumes.go | 1 + test/e2e/storage/drivers/csi.go | 36 ++++++++++++++----- test/e2e/storage/utils/deployment.go | 8 +++-- .../hostpath/csi-hostpath-driverinfo.yaml | 10 ++++++ 4 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 5d1d01c2028..706b9f7dd66 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -42,6 +42,7 @@ var csiTestDrivers = []func() testsuites.TestDriver{ // List of testSuites to be executed in below loop var csiTestSuites = []func() testsuites.TestSuite{ + testsuites.InitEphemeralTestSuite, testsuites.InitVolumesTestSuite, testsuites.InitVolumeIOTestSuite, testsuites.InitVolumeModeTestSuite, diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 87d511a45f3..e70dd468925 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -62,11 +62,12 @@ const ( // hostpathCSI type hostpathCSIDriver struct { - driverInfo testsuites.DriverInfo - manifests []string + driverInfo testsuites.DriverInfo + manifests []string + volumeAttributes []map[string]string } -func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver { +func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]bool, volumeAttributes []map[string]string, manifests ...string) testsuites.TestDriver { return &hostpathCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: name, @@ -77,13 +78,15 @@ func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]b ), Capabilities: capabilities, }, - manifests: manifests, + manifests: manifests, + volumeAttributes: volumeAttributes, } } var _ testsuites.TestDriver = &hostpathCSIDriver{} var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{} var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{} +var _ testsuites.EphemeralTestDriver = &hostpathCSIDriver{} // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface func InitHostPathCSIDriver() testsuites.TestDriver { @@ -97,15 +100,20 @@ func InitHostPathCSIDriver() testsuites.TestDriver { } return initHostPathCSIDriver("csi-hostpath", capabilities, + // Volume attributes don't matter, but we have to provide at least one map. + []map[string]string{ + {"foo": "bar"}, + }, "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml", - "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml", + "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml", ) } @@ -115,6 +123,9 @@ func (h *hostpathCSIDriver) GetDriverInfo() *testsuites.DriverInfo { } func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { + if pattern.VolType == testpatterns.CSIInlineVolume && len(h.volumeAttributes) == 0 { + framework.Skipf("%s has no volume attributes defined, doesn't support ephemeral inline volumes", h.driverInfo.Name) + } } func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass { @@ -126,6 +137,14 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.P return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } +func (h *hostpathCSIDriver) GetVolumeAttributes(config *testsuites.PerTestConfig, volumeNumber int) map[string]string { + return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)] +} + +func (h *hostpathCSIDriver) GetCSIDriverName(config *testsuites.PerTestConfig) string { + return config.GetUniqueDriverName() +} + func (h *hostpathCSIDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured { snapshotter := config.GetUniqueDriverName() parameters := map[string]string{} @@ -303,7 +322,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest NodeName: config.ClientNodeName, PodInfo: m.podInfo, CanAttach: &m.attachable, - VolumeLifecycleModes: []storagev1beta1.VolumeLifecycleMode{ + VolumeLifecycleModes: &[]storagev1beta1.VolumeLifecycleMode{ storagev1beta1.VolumeLifecyclePersistent, storagev1beta1.VolumeLifecycleEphemeral, }, @@ -327,6 +346,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest func InitHostPathV0CSIDriver() testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath-v0", map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true}, + nil, /* no volume attributes -> no ephemeral volume testing */ // Using the current set of rbac.yaml files is problematic here because they don't // match the version of the rules that were written for the releases of external-attacher // and external-provisioner that we are using here. It happens to work in practice... diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index 067e8cd20f4..c8161f7f057 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -131,7 +131,9 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf if o.CanAttach != nil { object.Spec.AttachRequired = o.CanAttach } - object.Spec.VolumeLifecycleModes = o.VolumeLifecycleModes + if o.VolumeLifecycleModes != nil { + object.Spec.VolumeLifecycleModes = *o.VolumeLifecycleModes + } } return nil @@ -171,8 +173,8 @@ type PatchCSIOptions struct { // field *if* the driver deploys a CSIDriver object. Ignored // otherwise. CanAttach *bool - // The value to use for the CSIDriver.Spec.VolumeLifecycleModes + // If not nil, the value to use for the CSIDriver.Spec.VolumeLifecycleModes // field *if* the driver deploys a CSIDriver object. Ignored // otherwise. - VolumeLifecycleModes []storagev1beta1.VolumeLifecycleMode + VolumeLifecycleModes *[]storagev1beta1.VolumeLifecycleMode } diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml new file mode 100644 index 00000000000..b4371003b34 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: hostpath.csi.k8s.io +spec: + # Supports both modes, but needs pod info for that to determine the actual mode. + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral From 55334fb9e9feac1e048078d5df3d3f48154a85df Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 30 Jul 2019 16:17:33 +0200 Subject: [PATCH 3/6] e2e storage: check NodeUnpublishVolume in csi-mock tests Once we have deleted the pod and the volume, we want to be sure that NodeUnpublishVolume was called for it. The main motivation was to check this for inline ephemeral volumes, but the same additional check also makes sense for other volumes. --- test/e2e/storage/csi_mock_volume.go | 42 +++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 4de7fa1f1ba..e127f01824f 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -370,10 +370,14 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { framework.ExpectNoError(err, "failed to test for CSIInlineVolumes") } + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting") + ginkgo.By("Checking CSI driver logs") // The driver is deployed as a statefulset with stable pod names driverPodName := "csi-mockplugin-0" - err = checkPodInfo(m.cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled) + err = checkPodLogs(m.cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled) framework.ExpectNoError(err) }) } @@ -719,8 +723,9 @@ func startPausePodWithVolumeSource(cs clientset.Interface, volumeSource v1.Volum return cs.CoreV1().Pods(ns).Create(pod) } -// checkPodInfo tests that NodePublish was called with expected volume_context -func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled bool) error { +// checkPodLogs tests that NodePublish was called with expected volume_context and (for ephemeral inline volumes) +// has the matching NodeUnpublish +func checkPodLogs(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled bool) error { expectedAttributes := map[string]string{ "csi.storage.k8s.io/pod.name": pod.Name, "csi.storage.k8s.io/pod.namespace": namespace, @@ -741,6 +746,8 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai // Find NodePublish in the logs foundAttributes := sets.NewString() logLines := strings.Split(log, "\n") + numNodePublishVolume := 0 + numNodeUnpublishVolume := 0 for _, line := range logLines { if !strings.HasPrefix(line, "gRPCCall:") { continue @@ -759,19 +766,23 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai e2elog.Logf("Could not parse CSI driver log line %q: %s", line, err) continue } - if call.Method != "/csi.v1.Node/NodePublishVolume" { - continue - } - // Check that NodePublish had expected attributes - for k, v := range expectedAttributes { - vv, found := call.Request.VolumeContext[k] - if found && v == vv { - foundAttributes.Insert(k) - e2elog.Logf("Found volume attribute %s: %s", k, v) + switch call.Method { + case "/csi.v1.Node/NodePublishVolume": + numNodePublishVolume++ + if numNodePublishVolume == 1 { + // Check that NodePublish had expected attributes for first volume + for k, v := range expectedAttributes { + vv, found := call.Request.VolumeContext[k] + if found && v == vv { + foundAttributes.Insert(k) + e2elog.Logf("Found volume attribute %s: %s", k, v) + } + } } + case "/csi.v1.Node/NodeUnpublishVolume": + e2elog.Logf("Found NodeUnpublishVolume: %+v", call) + numNodeUnpublishVolume++ } - // Process just the first NodePublish, the rest of the log is useless. - break } if expectPodInfo { if foundAttributes.Len() != len(expectedAttributes) { @@ -782,6 +793,9 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai if foundAttributes.Len() != 0 { return fmt.Errorf("some unexpected volume attributes were found: %+v", foundAttributes.List()) } + if numNodePublishVolume != numNodeUnpublishVolume { + return fmt.Errorf("number of NodePublishVolume %d != number of NodeUnpublishVolume %d", numNodePublishVolume, numNodeUnpublishVolume) + } return nil } From 7d2ad805a975b5a68e84a650aba92895441170b7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 30 Jul 2019 17:10:54 +0200 Subject: [PATCH 4/6] e2e storage: multi-volume and multi-pod tests for inline ephemeral volumes Multiple pods using the same data and multiple inline ephemeral volumes per pod are additional cases that are worth checking. --- test/e2e/storage/testsuites/ephemeral.go | 85 ++++++++++++++++++------ 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/test/e2e/storage/testsuites/ephemeral.go b/test/e2e/storage/testsuites/ephemeral.go index a1360dcfe24..097bf5a10e8 100644 --- a/test/e2e/storage/testsuites/ephemeral.go +++ b/test/e2e/storage/testsuites/ephemeral.go @@ -17,7 +17,9 @@ limitations under the License. package testsuites import ( + "flag" "fmt" + "strings" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -116,6 +118,35 @@ func (p *ephemeralTestSuite) defineTests(driver TestDriver, pattern testpatterns l.testCase.TestEphemeral() }) + + ginkgo.It("should support two pods which share the same data", func() { + init() + defer cleanup() + + l.testCase.RunningPodCheck = func(pod *v1.Pod) interface{} { + // Create another pod with the same inline volume attributes. + pod2 := StartInPodWithInlineVolume(f.ClientSet, f.Namespace.Name, "inline-volume-tester2", "true", + []v1.CSIVolumeSource{*pod.Spec.Volumes[0].CSI}, + l.testCase.Node) + defer StopPod(f.ClientSet, pod2) + framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceSlow(f.ClientSet, pod2.Name, pod2.Namespace), "waiting for second pod with inline volume") + return nil + } + + l.testCase.TestEphemeral() + }) + + var numInlineVolumes = flag.Int("storage.ephemeral."+strings.Replace(driver.GetDriverInfo().Name, ".", "-", -1)+".numInlineVolumes", + 2, "number of ephemeral inline volumes per pod") + + ginkgo.It("should support multiple inline ephemeral volumes", func() { + init() + defer cleanup() + + l.testCase.NumInlineVolumes = *numInlineVolumes + gomega.Expect(*numInlineVolumes).To(gomega.BeNumerically(">", 0), "positive number of inline volumes") + l.testCase.TestEphemeral() + }) } // EphemeralTest represents parameters to be used by tests for inline volumes. @@ -145,6 +176,10 @@ type EphemeralTest struct { // removed. How to do such a check is driver-specific and not // covered by the generic storage test suite. StoppedPodCheck func(nodeName string, runningPodData interface{}) + + // NumInlineVolumes sets the number of ephemeral inline volumes per pod. + // Unset (= zero) is the same as one. + NumInlineVolumes int } // TestEphemeral tests pod creation with one ephemeral volume. @@ -156,12 +191,18 @@ func (t EphemeralTest) TestEphemeral() { ginkgo.By(fmt.Sprintf("checking the requested inline volume exists in the pod running on node %+v", t.Node)) command := "mount | grep /mnt/test" - pod := StartInPodWithInlineVolume(client, t.Namespace, "inline-volume-tester", command, - v1.CSIVolumeSource{ + var csiVolumes []v1.CSIVolumeSource + numVolumes := t.NumInlineVolumes + if numVolumes == 0 { + numVolumes = 1 + } + for i := 0; i < numVolumes; i++ { + csiVolumes = append(csiVolumes, v1.CSIVolumeSource{ Driver: t.DriverName, - VolumeAttributes: t.GetVolumeAttributes(0), - }, - t.Node) + VolumeAttributes: t.GetVolumeAttributes(i), + }) + } + pod := StartInPodWithInlineVolume(client, t.Namespace, "inline-volume-tester", command, csiVolumes, t.Node) defer func() { // pod might be nil now. StopPod(client, pod) @@ -185,9 +226,9 @@ func (t EphemeralTest) TestEphemeral() { } } -// StartInPodWithInlineVolume starts a command in a pod with given volume mounted to /mnt/test directory. +// StartInPodWithInlineVolume starts a command in a pod with given volume(s) mounted to /mnt/test- directory. // The caller is responsible for checking the pod and deleting it. -func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command string, csiVolume v1.CSIVolumeSource, node e2epod.NodeSelection) *v1.Pod { +func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command string, csiVolumes []v1.CSIVolumeSource, node e2epod.NodeSelection) *v1.Pod { pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -208,26 +249,28 @@ func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command stri Name: "csi-volume-tester", Image: volume.GetTestImage(framework.BusyBoxImage), Command: volume.GenerateScriptCmd(command), - VolumeMounts: []v1.VolumeMount{ - { - Name: "my-volume", - MountPath: "/mnt/test", - }, - }, }, }, RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "my-volume", - VolumeSource: v1.VolumeSource{ - CSI: &csiVolume, - }, - }, - }, }, } + for i, csiVolume := range csiVolumes { + name := fmt.Sprintf("my-volume-%d", i) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, + v1.VolumeMount{ + Name: name, + MountPath: fmt.Sprintf("/mnt/test-%d", i), + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, + v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + CSI: &csiVolume, + }, + }) + } + pod, err := c.CoreV1().Pods(ns).Create(pod) framework.ExpectNoError(err, "failed to create pod") return pod From cead39cc97badc714fd51482a6936c3221e880c7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 30 Jul 2019 20:11:48 +0200 Subject: [PATCH 5/6] e2e storage: wait for pod deletion After deleting a pod, we need to be sure that it really is gone, otherwise there is a race condition: if we remove the CSI driver that is responsible for the volume used by the pod before the pod is actually deleted, deleting the pod will fail. --- test/e2e/storage/testsuites/provisioning.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 507e5677bc5..a93c416a5d5 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -588,7 +588,8 @@ func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command return pod } -// StopPod first tries to log the output of the pod's container, then deletes the pod. +// StopPod first tries to log the output of the pod's container, then deletes the pod and +// waits for that to succeed. func StopPod(c clientset.Interface, pod *v1.Pod) { if pod == nil { return @@ -600,6 +601,7 @@ func StopPod(c clientset.Interface, pod *v1.Pod) { e2elog.Logf("Pod %s has the following logs: %s", pod.Name, body) } e2epod.DeletePodOrFail(c, pod.Namespace, pod.Name) + e2epod.WaitForPodNoLongerRunningInNamespace(c, pod.Name, pod.Namespace) } func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeClaim) { From 5088b2ee2fbb99bdce42a19e59d9f172872ddf9a Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 31 Jul 2019 10:32:10 +0200 Subject: [PATCH 6/6] e2e storage: read/write + read-only ephemeral inline volume test, data sharing The assumption so far was that all drivers support read/write volumes. That might not necessarily be true, so we have to let the test driver specify it and then test accordingly. Another aspect that is worth testing is whether the driver correctly creates a new volume for each pod even if the volume attributes are the same. However, drivers are not required to do that, so again we have to let the test driver specify that. --- test/e2e/storage/drivers/csi.go | 4 +- test/e2e/storage/external/external.go | 30 ++++++--- test/e2e/storage/testsuites/ephemeral.go | 82 ++++++++++++++++++----- test/e2e/storage/testsuites/testdriver.go | 15 +++-- 4 files changed, 99 insertions(+), 32 deletions(-) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index e70dd468925..be4965c4d1f 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -137,8 +137,8 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.P return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) } -func (h *hostpathCSIDriver) GetVolumeAttributes(config *testsuites.PerTestConfig, volumeNumber int) map[string]string { - return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)] +func (h *hostpathCSIDriver) GetVolume(config *testsuites.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) { + return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)], false /* not shared */, false /* read-write */ } func (h *hostpathCSIDriver) GetCSIDriverName(config *testsuites.PerTestConfig) string { diff --git a/test/e2e/storage/external/external.go b/test/e2e/storage/external/external.go index 17035dd2d96..271f4d9a170 100644 --- a/test/e2e/storage/external/external.go +++ b/test/e2e/storage/external/external.go @@ -179,14 +179,25 @@ type driverDefinition struct { // TODO (?): load from file } - // InlineVolumeAttributes defines one or more set of attributes for - // use as inline ephemeral volumes. At least one set of attributes - // has to be defined to enable testing of inline ephemeral volumes. - // If a test needs more volumes than defined, some of the defined + // InlineVolumes defines one or more volumes for use as inline + // ephemeral volumes. At least one such volume has to be + // defined to enable testing of inline ephemeral volumes. If + // a test needs more volumes than defined, some of the defined // volumes will be used multiple times. // // DriverInfo.Name is used as name of the driver in the inline volume. - InlineVolumeAttributes []map[string]string + InlineVolumes []struct { + // Attributes are passed as NodePublishVolumeReq.volume_context. + // Can be empty. + Attributes map[string]string + // Shared defines whether the resulting volume is + // shared between different pods (i.e. changes made + // in one pod are visible in another) + Shared bool + // ReadOnly must be set to true if the driver does not + // support mounting as read/write. + ReadOnly bool + } // ClaimSize defines the desired size of dynamically // provisioned volumes. Default is "5GiB". @@ -221,7 +232,7 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern) supported = true } case testpatterns.CSIInlineVolume: - supported = len(d.InlineVolumeAttributes) != 0 + supported = len(d.InlineVolumes) != 0 } if !supported { framework.Skipf("Driver %q does not support volume type %q - skipping", d.DriverInfo.Name, pattern.VolType) @@ -294,11 +305,12 @@ func (d *driverDefinition) GetClaimSize() string { return d.ClaimSize } -func (d *driverDefinition) GetVolumeAttributes(config *testsuites.PerTestConfig, volumeNumber int) map[string]string { - if len(d.InlineVolumeAttributes) == 0 { +func (d *driverDefinition) GetVolume(config *testsuites.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) { + if len(d.InlineVolumes) == 0 { framework.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name) } - return d.InlineVolumeAttributes[volumeNumber%len(d.InlineVolumeAttributes)] + volume := d.InlineVolumes[volumeNumber%len(d.InlineVolumes)] + return volume.Attributes, volume.Shared, volume.ReadOnly } func (d *driverDefinition) GetCSIDriverName(config *testsuites.PerTestConfig) string { diff --git a/test/e2e/storage/testsuites/ephemeral.go b/test/e2e/storage/testsuites/ephemeral.go index 097bf5a10e8..53c5f70e900 100644 --- a/test/e2e/storage/testsuites/ephemeral.go +++ b/test/e2e/storage/testsuites/ephemeral.go @@ -32,6 +32,7 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/framework/volume" "k8s.io/kubernetes/test/e2e/storage/testpatterns" + storageutils "k8s.io/kubernetes/test/e2e/storage/utils" ) type ephemeralTestSuite struct { @@ -99,8 +100,8 @@ func (p *ephemeralTestSuite) defineTests(driver TestDriver, pattern testpatterns Namespace: f.Namespace.Name, DriverName: eDriver.GetCSIDriverName(l.config), Node: e2epod.NodeSelection{Name: l.config.ClientNodeName}, - GetVolumeAttributes: func(volumeNumber int) map[string]string { - return eDriver.GetVolumeAttributes(l.config, volumeNumber) + GetVolume: func(volumeNumber int) (map[string]string, bool, bool) { + return eDriver.GetVolume(l.config, volumeNumber) }, } } @@ -112,24 +113,58 @@ func (p *ephemeralTestSuite) defineTests(driver TestDriver, pattern testpatterns } } - ginkgo.It("should create inline ephemeral volume", func() { + ginkgo.It("should create read-only inline ephemeral volume", func() { init() defer cleanup() + l.testCase.ReadOnly = true + l.testCase.RunningPodCheck = func(pod *v1.Pod) interface{} { + storageutils.VerifyExecInPodSucceed(pod, "mount | grep /mnt/test | grep ro,") + return nil + } l.testCase.TestEphemeral() }) - ginkgo.It("should support two pods which share the same data", func() { + ginkgo.It("should create read/write inline ephemeral volume", func() { init() defer cleanup() + l.testCase.ReadOnly = false + l.testCase.RunningPodCheck = func(pod *v1.Pod) interface{} { + storageutils.VerifyExecInPodSucceed(pod, "mount | grep /mnt/test | grep rw,") + return nil + } + l.testCase.TestEphemeral() + }) + + ginkgo.It("should support two pods which share the same volume", func() { + init() + defer cleanup() + + // We test in read-only mode if that is all that the driver supports, + // otherwise read/write. + _, shared, readOnly := eDriver.GetVolume(l.config, 0) + l.testCase.RunningPodCheck = func(pod *v1.Pod) interface{} { // Create another pod with the same inline volume attributes. - pod2 := StartInPodWithInlineVolume(f.ClientSet, f.Namespace.Name, "inline-volume-tester2", "true", + pod2 := StartInPodWithInlineVolume(f.ClientSet, f.Namespace.Name, "inline-volume-tester2", "sleep 100000", []v1.CSIVolumeSource{*pod.Spec.Volumes[0].CSI}, + readOnly, l.testCase.Node) + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(f.ClientSet, pod2.Name, pod2.Namespace), "waiting for second pod with inline volume") + + // If (and only if) we were able to mount + // read/write and volume data is not shared + // between pods, then we can check whether + // data written in one pod is really not + // visible in the other. + if !readOnly && !shared { + ginkgo.By("writing data in one pod and checking for it in the second") + storageutils.VerifyExecInPodSucceed(pod, "touch /mnt/test-0/hello-world") + storageutils.VerifyExecInPodSucceed(pod2, "[ ! -f /mnt/test-0/hello-world ]") + } + defer StopPod(f.ClientSet, pod2) - framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceSlow(f.ClientSet, pod2.Name, pod2.Namespace), "waiting for second pod with inline volume") return nil } @@ -157,12 +192,18 @@ type EphemeralTest struct { DriverName string Node e2epod.NodeSelection - // GetVolumeAttributes returns the volume attributes for a + // GetVolume returns the volume attributes for a // certain inline ephemeral volume, enumerated starting with // #0. Some tests might require more than one volume. They can // all be the same or different, depending what the driver supports // and/or wants to test. - GetVolumeAttributes func(volumeNumber int) map[string]string + // + // For each volume, the test driver can specify the + // attributes, whether two pods using those attributes will + // end up sharing the same backend storage (i.e. changes made + // in one pod will be visible in the other), and whether + // the volume can be mounted read/write or only read-only. + GetVolume func(volumeNumber int) (attributes map[string]string, shared bool, readOnly bool) // RunningPodCheck is invoked while a pod using an inline volume is running. // It can execute additional checks on the pod and its volume(s). Any data @@ -180,34 +221,42 @@ type EphemeralTest struct { // NumInlineVolumes sets the number of ephemeral inline volumes per pod. // Unset (= zero) is the same as one. NumInlineVolumes int + + // ReadOnly limits mounting to read-only. + ReadOnly bool } // TestEphemeral tests pod creation with one ephemeral volume. func (t EphemeralTest) TestEphemeral() { client := t.Client gomega.Expect(client).NotTo(gomega.BeNil(), "EphemeralTest.Client is required") - gomega.Expect(t.GetVolumeAttributes).NotTo(gomega.BeNil(), "EphemeralTest.GetVolumeAttributes is required") + gomega.Expect(t.GetVolume).NotTo(gomega.BeNil(), "EphemeralTest.GetVolume is required") gomega.Expect(t.DriverName).NotTo(gomega.BeEmpty(), "EphemeralTest.DriverName is required") ginkgo.By(fmt.Sprintf("checking the requested inline volume exists in the pod running on node %+v", t.Node)) - command := "mount | grep /mnt/test" + command := "mount | grep /mnt/test && sleep 10000" var csiVolumes []v1.CSIVolumeSource numVolumes := t.NumInlineVolumes if numVolumes == 0 { numVolumes = 1 } for i := 0; i < numVolumes; i++ { - csiVolumes = append(csiVolumes, v1.CSIVolumeSource{ + attributes, _, readOnly := t.GetVolume(i) + csi := v1.CSIVolumeSource{ Driver: t.DriverName, - VolumeAttributes: t.GetVolumeAttributes(i), - }) + VolumeAttributes: attributes, + } + if readOnly && !t.ReadOnly { + framework.Skipf("inline ephemeral volume #%d is read-only, but the test needs a read/write volume", i) + } + csiVolumes = append(csiVolumes, csi) } - pod := StartInPodWithInlineVolume(client, t.Namespace, "inline-volume-tester", command, csiVolumes, t.Node) + pod := StartInPodWithInlineVolume(client, t.Namespace, "inline-volume-tester", command, csiVolumes, t.ReadOnly, t.Node) defer func() { // pod might be nil now. StopPod(client, pod) }() - framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceSlow(client, pod.Name, pod.Namespace), "waiting for pod with inline volume") + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(client, pod.Name, pod.Namespace), "waiting for pod with inline volume") runningPod, err := client.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "get pod") actualNodeName := runningPod.Spec.NodeName @@ -228,7 +277,7 @@ func (t EphemeralTest) TestEphemeral() { // StartInPodWithInlineVolume starts a command in a pod with given volume(s) mounted to /mnt/test- directory. // The caller is responsible for checking the pod and deleting it. -func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command string, csiVolumes []v1.CSIVolumeSource, node e2epod.NodeSelection) *v1.Pod { +func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command string, csiVolumes []v1.CSIVolumeSource, readOnly bool, node e2epod.NodeSelection) *v1.Pod { pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -261,6 +310,7 @@ func StartInPodWithInlineVolume(c clientset.Interface, ns, podName, command stri v1.VolumeMount{ Name: name, MountPath: fmt.Sprintf("/mnt/test-%d", i), + ReadOnly: readOnly, }) pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ diff --git a/test/e2e/storage/testsuites/testdriver.go b/test/e2e/storage/testsuites/testdriver.go index e7d060ebee7..34ea0085fdc 100644 --- a/test/e2e/storage/testsuites/testdriver.go +++ b/test/e2e/storage/testsuites/testdriver.go @@ -105,12 +105,17 @@ type DynamicPVTestDriver interface { type EphemeralTestDriver interface { TestDriver - // GetVolumeAttributes returns the volume attributes for a - // certain inline ephemeral volume, enumerated starting with - // #0. Some tests might require more than one volume. They can - // all be the same or different, depending what the driver supports + // GetVolume returns the volume attributes for a certain + // inline ephemeral volume, enumerated starting with #0. Some + // tests might require more than one volume. They can all be + // the same or different, depending what the driver supports // and/or wants to test. - GetVolumeAttributes(config *PerTestConfig, volumeNumber int) map[string]string + // + // For each volume, the test driver can return volume attributes, + // whether the resulting volume is shared between different pods (i.e. + // changes made in one pod are visible in another), and whether the + // volume can be mounted read/write or only read-only. + GetVolume(config *PerTestConfig, volumeNumber int) (attributes map[string]string, shared bool, readOnly bool) // GetCSIDriverName returns the name that was used when registering with // kubelet. Depending on how the driver was deployed, this can be different