From 446c1136dc526004686a9d1da5dad6d665554409 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 29 Mar 2021 18:49:32 +0200 Subject: [PATCH 1/4] storage e2e: automate hostpath YAML updates, hostpath v1.6.2 Mirroring the various YAML files by hand is tedious. The new update-hostpath.sh does all the necessary steps automatically. The result is now a bit more consistent with the upstream repos in the sense that the original file names and paths for the RBAC YAML files are used. The csi-hostpath-testing.yaml is included for the sake of completeness, but not used during E2E testing. The new hostpath driver release is v1.6.2, which adds the external-health-monitor for the first time. --- test/e2e/storage/drivers/csi.go | 25 +++- .../storage-csi/external-attacher/README.md | 1 - .../storage-csi/external-attacher/rbac.yaml | 13 +- .../external-health-monitor-agent/rbac.yaml | 60 ++++++++ .../rbac.yaml | 89 ++++++++++++ .../external-provisioner/README.md | 1 - .../external-provisioner/rbac.yaml | 26 ++++ .../storage-csi/external-resizer/README.md | 1 - .../storage-csi/external-resizer/rbac.yaml | 11 +- .../external-snapshotter/README.md | 1 - .../rbac-csi-snapshotter.yaml} | 14 +- .../storage-csi/hostpath/README.md | 9 +- .../hostpath/csi-hostpath-attacher.yaml | 16 +-- .../hostpath/csi-hostpath-plugin.yaml | 39 +++++- .../hostpath/csi-hostpath-provisioner.yaml | 16 +-- .../hostpath/csi-hostpath-resizer.yaml | 16 +-- .../hostpath/csi-hostpath-snapshotter.yaml | 16 +-- .../hostpath/csi-hostpath-testing.yaml | 64 +++++++++ .../hostpath/hostpath/e2e-test-rbac.yaml | 3 + .../hostpath/usage/csi-storageclass.yaml | 7 - .../storage-csi/update-hostpath.sh | 128 ++++++++++++++++++ 21 files changed, 456 insertions(+), 100 deletions(-) delete mode 100644 test/e2e/testing-manifests/storage-csi/external-attacher/README.md create mode 100644 test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml create mode 100644 test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml delete mode 100644 test/e2e/testing-manifests/storage-csi/external-provisioner/README.md delete mode 100644 test/e2e/testing-manifests/storage-csi/external-resizer/README.md delete mode 100644 test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md rename test/e2e/testing-manifests/storage-csi/external-snapshotter/{rbac.yaml => csi-snapshotter/rbac-csi-snapshotter.yaml} (79%) create mode 100644 test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml delete mode 100644 test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml create mode 100755 test/e2e/testing-manifests/storage-csi/update-hostpath.sh diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 2035c8d6a11..70eba6b2e70 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -142,7 +142,12 @@ func InitHostPathCSIDriver() storageframework.TestDriver { storageframework.CapPVCDataSource: true, storageframework.CapControllerExpansion: true, storageframework.CapSingleNodeVolume: true, - storageframework.CapVolumeLimits: true, + + // This is needed for the + // testsuites/volumelimits.go `should support volume limits` + // test. --maxvolumespernode=10 gets + // added when patching the deployment. + storageframework.CapVolumeLimits: true, } return initHostPathCSIDriver("csi-hostpath", capabilities, @@ -152,7 +157,8 @@ func InitHostPathCSIDriver() storageframework.TestDriver { }, "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml", + "test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml", @@ -220,10 +226,15 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo } o := utils.PatchCSIOptions{ - OldDriverName: h.driverInfo.Name, - NewDriverName: config.GetUniqueDriverName(), - DriverContainerName: "hostpath", - DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName()}, + OldDriverName: h.driverInfo.Name, + NewDriverName: config.GetUniqueDriverName(), + DriverContainerName: "hostpath", + DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(), + // This is needed for the + // testsuites/volumelimits.go `should support volume limits` + // test. + "--maxvolumespernode=10", + }, ProvisionerContainerName: "csi-provisioner", SnapshotterContainerName: "csi-snapshotter", NodeName: node.Name, @@ -408,7 +419,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver { "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", } diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md b/test/e2e/testing-manifests/storage-csi/external-attacher/README.md deleted file mode 100644 index a8766137cd0..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-attacher/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml index 4d6dd506ce1..9111dc30c77 100644 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-attacher/raw/v3.1.0/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI attacher. # @@ -16,7 +20,7 @@ metadata: namespace: default --- -# Attacher must be able to work with PVs, nodes and VolumeAttachments +# Attacher must be able to work with PVs, CSINodes and VolumeAttachments kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -24,16 +28,13 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] diff --git a/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml new file mode 100644 index 00000000000..f698e8fdaff --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml @@ -0,0 +1,60 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-agent/rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# +# This YAML file contains all RBAC objects that are necessary to run external +# CSI health monitor agent. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# health monitor agent, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-external-health-monitor-agent + # replace with non-default namespace name + namespace: default + +--- +# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-health-monitor-agent-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-agent-role +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-agent + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-health-monitor-agent-runner + apiGroup: rbac.authorization.k8s.io + + diff --git a/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml new file mode 100644 index 00000000000..1ec35db8cf5 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml @@ -0,0 +1,89 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-controller/rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# +# This YAML file contains all RBAC objects that are necessary to run external +# CSI health monitor controller. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# health monitor controller, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default + +--- +# Health monitor controller must be able to work with PVs, PVCs, Nodes and Pods +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-health-monitor-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-controller-role +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-health-monitor-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +# Health monitor controller must be able to work with configmaps or leases in the current namespace +# if (and only if) leadership election is enabled +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: external-health-monitor-controller-cfg +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-controller-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: external-health-monitor-controller-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md b/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md deleted file mode 100644 index 982a1adae98..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-provisioner/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml index 35b68801595..68cc4c1dfac 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-provisioner/raw/v2.1.1/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI provisioner. # @@ -50,6 +54,13 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -84,6 +95,21 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +# Permissions for CSIStorageCapacity are only needed enabling the publishing +# of storage capacity information. +- apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# The GET permissions below are needed for walking up the ownership chain +# for CSIStorageCapacity. They are sufficient for deployment via +# StatefulSet (only needs to get Pod) and Deployment (needs to get +# Pod and then ReplicaSet to find the Deployment). +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +- apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] --- kind: RoleBinding diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md deleted file mode 100644 index 4e7e248a7ff..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-resizer/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml index 17ed01f8381..590c5420836 100644 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-resizer/raw/v1.1.0/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI resizer. # @@ -29,13 +33,16 @@ rules: # verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] + verbs: ["patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md deleted file mode 100644 index d96fc148794..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml similarity index 79% rename from test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml rename to test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml index ca4a54b6e05..6577637c773 100644 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-snapshotter/raw/v4.0.0/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # Together with the RBAC file for external-provisioner, this YAML file # contains all RBAC objects that are necessary to run external CSI # snapshotter. @@ -23,9 +27,13 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/README.md b/test/e2e/testing-manifests/storage-csi/hostpath/README.md index c2990622700..2f06c7e550a 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/README.md +++ b/test/e2e/testing-manifests/storage-csi/hostpath/README.md @@ -1,5 +1,4 @@ -A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example, -with some modifications: -- serviceAccountName is used instead of the deprecated serviceAccount -- the RBAC roles from driver-registrar, external-attacher, external-provisioner - and external-snapshotter are used +The files in this directory are exact copys of "kubernetes-latest" in +https://github.com/kubernetes-csi/csi-driver-host-path/tree/v1.6.2/deploy/ + +Do not edit manually. Run test/e2e/testing-manifests/storage-csi/update-hostpath.sh to refresh the content. diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml index 2c23f75c71c..6c5a14a146c 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-attacher - labels: - app: csi-hostpath-attacher -spec: - selector: - app: csi-hostpath-attacher - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-attacher containers: - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v2.2.0 + image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0 args: - --v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 5997d00e71d..9e382bbe201 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -34,9 +34,39 @@ spec: labels: app: csi-hostpathplugin spec: + serviceAccount: csi-external-health-monitor-controller containers: + - name: csi-external-health-monitor-agent + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-external-health-monitor-controller + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-controller:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: node-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.3.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 args: - --v=5 - --csi-address=/csi/csi.sock @@ -61,15 +91,12 @@ spec: name: csi-data-dir - name: hostpath - image: k8s.gcr.io/sig-storage/hostpathplugin:v1.4.0 + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 args: - "--drivername=hostpath.csi.k8s.io" - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - # The only difference to github.com/kubernetes-csi/csi-driver-host-path/deploy - # - we have a tests that checks node limits. - - "--maxvolumespernode=10" env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock @@ -109,7 +136,7 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - image: k8s.gcr.io/sig-storage/livenessprobe:v1.1.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 args: - --csi-address=/csi/csi.sock - --health-port=9898 diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml index 856dbff120c..0cbc9910cea 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-provisioner - labels: - app: csi-hostpath-provisioner -spec: - selector: - app: csi-hostpath-provisioner - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-provisioner containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v1.6.0 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.1.1 args: - -v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml index 0294f8b2a38..524f9ed4f36 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-resizer - labels: - app: csi-hostpath-resizer -spec: - selector: - app: csi-hostpath-resizer - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-resizer containers: - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v0.5.0 + image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 args: - -v=5 - -csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml index 66c3882ff64..3a61d9e14e4 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-snapshotter - labels: - app: csi-hostpath-snapshotter -spec: - selector: - app: csi-hostpath-snapshotter - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccount: csi-snapshotter containers: - name: csi-snapshotter - image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2 + image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0 args: - -v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 00000000000..188a5bde870 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,64 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service +spec: + type: NodePort + selector: + app: csi-hostpath-socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-socat + template: + metadata: + labels: + app: csi-hostpath-socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml index dde7ce78258..3cd01df1f74 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml @@ -16,6 +16,9 @@ subjects: - kind: ServiceAccount name: csi-resizer namespace: default + - kind: ServiceAccount + name: csi-external-health-monitor-controller + namespace: default roleRef: kind: ClusterRole name: e2e-test-privileged-psp diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml deleted file mode 100644 index c92797167e6..00000000000 --- a/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-hostpath-sc -provisioner: csi-hostpath -reclaimPolicy: Delete -volumeBindingMode: Immediate diff --git a/test/e2e/testing-manifests/storage-csi/update-hostpath.sh b/test/e2e/testing-manifests/storage-csi/update-hostpath.sh new file mode 100755 index 00000000000..493baedaa25 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/update-hostpath.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will update all sidecar RBAC files and the CSI hostpath +# deployment files such that they match what is in a hostpath driver +# release. +# +# Beware that this will wipe out all local modifications! + +# Can be a tag or a branch. +script="$0" +hostpath_version="$1" + +if ! [ "$hostpath_version" ]; then + cat >&2 < + +Required parameter is missing. +EOF + exit 1 +fi + +set -xe +cd "$(dirname "$0")" + +# Remove stale files. +rm -rf external-attacher external-provisioner external-resizer external-snapshotter external-health-monitor hostpath csi-driver-host-path + +# Check out desired release. +git clone https://github.com/kubernetes-csi/csi-driver-host-path.git +(cd csi-driver-host-path && git checkout "$hostpath_version") +trap "rm -rf csi-driver-host-path" EXIT + +# Main YAML files. +mkdir hostpath +cat >hostpath/README.md <hostpath/hostpath/e2e-test-rbac.yaml <"$project/$path/$rbac" <>"$project/$path/$rbac" +} + +# RBAC files for each sidecar. +# This relies on the convention that "external-something" has "csi-something" as image name. +# external-health-monitor is special, it has two images. +# The repository for each image is ignored. +images=$(grep -r '^ *image:.*csi' hostpath/hostpath | sed -e 's;.*image:.*/;;' | grep -v 'node-driver-registrar' | sort -u) +for image in $images; do + tag=$(echo "$image" | sed -e 's/.*://') + path= + rbac="rbac.yaml" + case $image in + csi-external-*) + # csi-external-health-monitor-agent:v0.2.0 + project=$(echo "$image" | sed -e 's/csi-\(.*\)-[^:]*:.*/\1/') + path=$(echo "$image" | sed -e 's/csi-\([^:]*\):.*/\1/') + ;; + *) + project=$(echo "$image" | sed -e 's/:.*//' -e 's/^csi/external/') + case $project in + external-snapshotter) + # Another special case... + path="csi-snapshotter" + rbac="rbac-csi-snapshotter.yaml" + ;; + esac + ;; + esac + download "$project" "$path" "$tag" "$rbac" +done From 7682e39a4717b61864708eecfb7fb7772a6eeb42 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 1 Apr 2021 08:47:09 +0200 Subject: [PATCH 2/4] storage e2e: disable health check containers They are not needed for any of the tests and in practice apparently caused enough overhead that even unrelated tests timed out. For example, in the pull-kubernetes-e2e-kind test, 43 out of 5771 tests failed, including tests from sig-node, sig-cli, sig-api-machinery, sig-network. --- test/e2e/storage/drivers/csi.go | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 70eba6b2e70..beb33c1c9f0 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -47,6 +47,7 @@ import ( "github.com/onsi/ginkgo" "google.golang.org/grpc/codes" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -240,7 +241,33 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo NodeName: node.Name, } cleanup, err := utils.CreateFromManifests(config.Framework, driverNamespace, func(item interface{}) error { - return utils.PatchCSIDeployment(config.Framework, o, item) + if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { + return err + } + + // Remove csi-external-health-monitor-agent and + // csi-external-health-monitor-controller + // containers. They are not needed for any of the + // tests and in practice apparently caused enough + // overhead that even unrelated tests timed out. For + // example, in the pull-kubernetes-e2e-kind test, 43 + // out of 5771 tests failed, including tests from + // sig-node, sig-cli, sig-api-machinery, sig-network. + switch item := item.(type) { + case *appsv1.StatefulSet: + var containers []v1.Container + for _, container := range item.Spec.Template.Spec.Containers { + switch container.Name { + case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller": + // Remove these containers. + default: + // Keep the others. + containers = append(containers, container) + } + } + item.Spec.Template.Spec.Containers = containers + } + return nil }, h.manifests...) if err != nil { From ebd02341c9805086ffc0af1422ed6133b86142f1 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 20 Apr 2021 08:06:07 +0200 Subject: [PATCH 3/4] storage e2e: downgrade hostpath driver This is a temporary workaround until a fixed driver is available. --- .../storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 9e382bbe201..5744ea6186e 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -91,7 +91,10 @@ spec: name: csi-data-dir - name: hostpath - image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 + # WARNING: manually downgraded from 1.6.2 to 1.4.0 because 1.5.x and 1.6.x have + # a bug that causes E2E testing to fail (https://github.com/kubernetes-csi/csi-driver-host-path/pull/210#discussion_r605592438, + # https://github.com/kubernetes-csi/csi-driver-host-path/issues/251). + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.4.0 args: - "--drivername=hostpath.csi.k8s.io" - "--v=5" From c794b5c442b18b078b287bcbb25bb18b9eb439cd Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 20 Apr 2021 20:57:10 +0200 Subject: [PATCH 4/4] storage e2e: patch in RBAC rules for secrets In one mock test, the snapshotter needs permission to read secrets. That was disabled in the RBAC files of recent releases. We need to patch it back in during deployment. --- test/e2e/storage/drivers/csi.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index beb33c1c9f0..c5775767f36 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -49,6 +49,7 @@ import ( "google.golang.org/grpc/codes" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -649,7 +650,25 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.P FSGroupPolicy: m.fsGroupPolicy, } cleanup, err := utils.CreateFromManifests(f, m.driverNamespace, func(item interface{}) error { - return utils.PatchCSIDeployment(f, o, item) + if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { + return err + } + + switch item := item.(type) { + case *rbacv1.ClusterRole: + if strings.HasPrefix(item.Name, "external-snapshotter-runner") { + // Re-enable access to secrets for the snapshotter sidecar for + // https://github.com/kubernetes/kubernetes/blob/6ede5ca95f78478fa627ecfea8136e0dff34436b/test/e2e/storage/csi_mock_volume.go#L1539-L1548 + // It was disabled in https://github.com/kubernetes-csi/external-snapshotter/blob/501cc505846c03ee665355132f2da0ce7d5d747d/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml#L26-L32 + item.Rules = append(item.Rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list"}, + }) + } + } + + return nil }, m.manifests...) if err != nil {