e2e storage: synchronize .yaml files with latest upstream releases

This ensures that the files are in sync with:
   hostpath: v1.2.0-rc3
   external-attacher: v2.0.1
   external-provisioner: v1.3.0
   external-resizer: v0.2.0
   external-snapshotter: v1.2.0

driver-registrar/rbac.yaml is obsolete because only
node-driver-registrar is in use now and does not need RBAC rules.

mock/e2e-test-rbac.yaml was not used anywhere.

The README.md files were updated to indicate that these really are
files copied from elsewhere. To avoid the need to constantly edit
these files on each update, <version> is used as placeholder in the URL.

test/e2e/storage/drivers/csi.go

@@ -97,7 +97,6 @@ func InitHostPathCSIDriver() testsuites.TestDriver {
 	}
 	return initHostPathCSIDriver("csi-hostpath",
 		capabilities,
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
@@ -106,7 +105,7 @@ func InitHostPathCSIDriver() testsuites.TestDriver {
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml",
-		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
 	)
 }
@@ -205,7 +204,6 @@ var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{}
 // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface
 func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver {
 	driverManifests := []string{
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
@@ -329,7 +327,9 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 func InitHostPathV0CSIDriver() testsuites.TestDriver {
 	return initHostPathCSIDriver("csi-hostpath-v0",
 		map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true},
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		// Using the current set of rbac.yaml files is problematic here because they don't
+		// match the version of the rules that were written for the releases of external-attacher
+		// and external-provisioner that we are using here. It happens to work in practice...
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-attacher.yaml",
@@ -423,7 +423,6 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
 	createGCESecrets(f.ClientSet, f.Namespace.Name)
 
 	manifests := []string{
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml",

test/e2e/testing-manifests/storage-csi/driver-registrar/README.md

@@ -1 +0,0 @@
-The original file is (or will be) https://github.com/kubernetes-csi/driver-registrar/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml

@@ -1,51 +0,0 @@
-# This YAML file contains all RBAC objects that are necessary to run external
-# CSI provisioner.
-#
-# In production, each CSI driver deployment has to be customized:
-# - to avoid conflicts, use non-default namespace and different names
-#   for non-namespaced entities like the ClusterRole
-# - decide whether the deployment replicates the external CSI
-#   provisioner, in which case leadership election must be enabled;
-#   this influences the RBAC setup, see below
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: csi-node-sa
-  # replace with non-default namespace name
-  namespace: default
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: driver-registrar-runner
-rules:
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
-  # The following permissions are only needed when running
-  # driver-registrar without the --kubelet-registration-path
-  # parameter, i.e. when using driver-registrar instead of
-  # kubelet to update the csi.volume.kubernetes.io/nodeid
-  # annotation. That mode of operation is going to be deprecated
-  # and should not be used anymore, but is needed on older
-  # Kubernetes versions.
-  # - apiGroups: [""]
-  #   resources: ["nodes"]
-  #   verbs: ["get", "update", "patch"]
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: csi-driver-registrar-role
-subjects:
-  - kind: ServiceAccount
-    name: csi-node-sa
-    # replace with non-default namespace name
-    namespace: default
-roleRef:
-  kind: ClusterRole
-  name: driver-registrar-runner
-  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/external-attacher/README.md

@@ -1 +1 @@
-The original file is (or will be) https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
+The original file is https://github.com/kubernetes-csi/external-attacher/blob/<version>/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-provisioner/README.md

@@ -1 +1 @@
-The original file is (or will be) https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
+The original file is https://github.com/kubernetes-csi/external-provisioner/blob/<version>/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-resizer/README.md

@@ -1 +1 @@
-The original file is (or will be) https://github.com/kubernetes-csi/external-resizer/blob/master/deploy/kubernetes/rbac.yaml
+The original file is https://github.com/kubernetes-csi/external-resizer/blob/<version>/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml

@@ -36,9 +36,6 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -59,7 +56,7 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 
 ---
-# Resizer must be able to work with leases in current namespace
+# Resizer must be able to work with end point in current namespace
 # if (and only if) leadership election is enabled
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md

@@ -1 +1 @@
-The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml
+The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/<version>/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml

@@ -20,36 +20,36 @@ metadata:
   # rename if there are conflicts
   name: external-snapshotter-runner
 rules:
-- apiGroups: [""]
+  - apiGroups: [""]
-  resources: ["persistentvolumes"]
+    resources: ["persistentvolumes"]
-  verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch"]
-- apiGroups: [""]
+  - apiGroups: [""]
-  resources: ["persistentvolumeclaims"]
+    resources: ["persistentvolumeclaims"]
-  verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update"]
-- apiGroups: ["storage.k8s.io"]
+  - apiGroups: ["storage.k8s.io"]
-  resources: ["storageclasses"]
+    resources: ["storageclasses"]
-  verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch"]
-- apiGroups: [""]
+  - apiGroups: [""]
-  resources: ["events"]
+    resources: ["events"]
-  verbs: ["list", "watch", "create", "update", "patch"]
+    verbs: ["list", "watch", "create", "update", "patch"]
-- apiGroups: [""]
+  - apiGroups: [""]
-  resources: ["secrets"]
+    resources: ["secrets"]
-  verbs: ["get", "list"]
+    verbs: ["get", "list"]
-- apiGroups: ["snapshot.storage.k8s.io"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["volumesnapshotclasses"]
+    resources: ["volumesnapshotclasses"]
-  verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch"]
-- apiGroups: ["snapshot.storage.k8s.io"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["volumesnapshotcontents"]
+    resources: ["volumesnapshotcontents"]
-  verbs: ["create", "get", "list", "watch", "update", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
-- apiGroups: ["snapshot.storage.k8s.io"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["volumesnapshots"]
+    resources: ["volumesnapshots"]
-  verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update"]
-- apiGroups: ["snapshot.storage.k8s.io"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["volumesnapshots/status"]
+    resources: ["volumesnapshots/status"]
-  verbs: ["update"]
+    verbs: ["update"]
-- apiGroups: ["apiextensions.k8s.io"]
+  - apiGroups: ["apiextensions.k8s.io"]
-  resources: ["customresourcedefinitions"]
+    resources: ["customresourcedefinitions"]
-  verbs: ["create", "list", "watch", "delete"]
+    verbs: ["create", "list", "watch", "delete", "get", "update"]
 
 ---
 kind: ClusterRoleBinding
@@ -57,12 +57,39 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: csi-snapshotter-role
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
-  name: csi-snapshotter
+    name: csi-snapshotter
-  # replace with non-default namespace name
+    # replace with non-default namespace name
-  namespace: default
+    namespace: default
 roleRef:
   kind: ClusterRole
   # change the name also here if the ClusterRole gets renamed
   name: external-snapshotter-runner
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default # TODO: replace with the namespace you want for your sidecar
+  name: external-snapshotter-leaderelection
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-snapshotter-leaderelection
+  namespace: default # TODO: replace with the namespace you want for your sidecar
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter
+    namespace: default # TODO: replace with the namespace you want for your sidecar
+roleRef:
+  kind: Role
+  name: external-snapshotter-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+

test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml

@@ -69,9 +69,6 @@ subjects:
   - kind: ServiceAccount
     name: csi-controller-sa
     namespace: default
-  - kind: ServiceAccount
-    name: csi-node-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: e2e-test-privileged-psp

test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml

@@ -11,7 +11,6 @@ spec:
       labels:
         app: gcp-compute-persistent-disk-csi-driver
     spec:
-      serviceAccountName: csi-node-sa
       containers:
         - name: csi-driver-registrar
           image: gcr.io/gke-release/csi-node-driver-registrar:v1.1.0-gke.0

test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpathplugin.yaml

@@ -11,7 +11,6 @@ spec:
       labels:
         app: csi-hostpathplugin
     spec:
-      serviceAccountName: csi-node-sa
       hostNetwork: true
       containers:
         - name: driver-registrar

test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/e2e-test-rbac.yaml

@@ -7,9 +7,6 @@ subjects:
   - kind: ServiceAccount
     name: csi-attacher
     namespace: default
-  - kind: ServiceAccount
-    name: csi-node-sa
-    namespace: default
   - kind: ServiceAccount
     name: csi-provisioner
     namespace: default

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml

@@ -27,20 +27,27 @@ spec:
       labels:
         app: csi-hostpath-attacher
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
       serviceAccountName: csi-attacher
       containers:
         - name: csi-attacher
           image: quay.io/k8scsi/csi-attacher:v1.2.0
           args:
             - --v=5
-            - --csi-address=$(ADDRESS)
+            - --csi-address=/csi/csi.sock
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          imagePullPolicy: Always
           volumeMounts:
           - mountPath: /csi
             name: socket-dir
+
       volumes:
         - hostPath:
             path: /var/lib/kubelet/plugins/csi-hostpath

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml

@@ -1,8 +1,31 @@
-kind: DaemonSet
+# Service defined here, plus serviceName below in StatefulSet,
+# are needed only because of condition explained in
+# https://github.com/kubernetes/kubernetes/issues/69608
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpathplugin
+  labels:
+    app: csi-hostpathplugin
+spec:
+  selector:
+    app: csi-hostpathplugin
+  ports:
+    - name: dummy
+      port: 12345
+---
+kind: StatefulSet
 apiVersion: apps/v1
 metadata:
   name: csi-hostpathplugin
 spec:
+  serviceName: "csi-hostpathplugin"
+  # One replica only:
+  # Host path driver only works when everything runs
+  # on a single node. We achieve that by starting it once and then
+  # co-locate all other pods via inter-pod affinity
+  replicas: 1
   selector:
     matchLabels:
       app: csi-hostpathplugin
@@ -11,30 +34,38 @@ spec:
       labels:
         app: csi-hostpathplugin
     spec:
-      serviceAccountName: csi-node-sa
       hostNetwork: true
       containers:
-        - name: driver-registrar
+        - name: node-driver-registrar
           image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"]
           args:
             - --v=5
             - --csi-address=/csi/csi.sock
             - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+          securityContext:
+            privileged: true
           env:
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: spec.nodeName
-          imagePullPolicy: Always
           volumeMounts:
           - mountPath: /csi
             name: socket-dir
           - mountPath: /registration
             name: registration-dir
+          - mountPath: /csi-data-dir
+            name: csi-data-dir
+
         - name: hostpath
           image: quay.io/k8scsi/hostpathplugin:v1.2.0-rc5
           args:
+            - "--drivername=hostpath.csi.k8s.io"
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(KUBE_NODE_NAME)"
@@ -46,9 +77,20 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: spec.nodeName
-          imagePullPolicy: Always
           securityContext:
             privileged: true
+          ports:
+          - containerPort: 9898
+            name: healthz
+            protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 2
           volumeMounts:
             - mountPath: /csi
               name: socket-dir
@@ -58,6 +100,19 @@ spec:
             - mountPath: /var/lib/kubelet/plugins
               mountPropagation: Bidirectional
               name: plugins-dir
+            - mountPath: /csi-data-dir
+              name: csi-data-dir
+
+        - name: liveness-probe
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          image: quay.io/k8scsi/livenessprobe:v1.1.0
+          args:
+          - --csi-address=/csi/csi.sock
+          - --connection-timeout=3s
+          - --health-port=9898
+
       volumes:
         - hostPath:
             path: /var/lib/kubelet/plugins/csi-hostpath
@@ -75,3 +130,9 @@ spec:
             path: /var/lib/kubelet/plugins
             type: Directory
           name: plugins-dir
+        - hostPath:
+            # 'path' is where PV data is persisted on host.
+            # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot
+            path: /var/lib/csi-hostpath-data/
+            type: DirectoryOrCreate
+          name: csi-data-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml

@@ -1,12 +1,12 @@
 kind: Service
 apiVersion: v1
 metadata:
-  name: csi-hostpath-provisioner
+  name: csi-hostpath-provisioner 
   labels:
-    app: csi-hostpath-provisioner
+    app: csi-hostpath-provisioner 
 spec:
   selector:
-    app: csi-hostpath-provisioner
+    app: csi-hostpath-provisioner 
   ports:
     - name: dummy
       port: 12345
@@ -27,19 +27,25 @@ spec:
       labels:
         app: csi-hostpath-provisioner
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
       serviceAccountName: csi-provisioner
       containers:
         - name: csi-provisioner
           # TODO: replace with official 1.4.0 release when ready
           image: quay.io/k8scsi/csi-provisioner:v1.4.0-rc1
           args:
-            - "--provisioner=csi-hostpath"
+            - -v=5
-            - "--csi-address=$(ADDRESS)"
+            - --csi-address=/csi/csi.sock
-            - "--connection-timeout=15s"
+            - --connection-timeout=15s
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          imagePullPolicy: Always
           volumeMounts:
             - mountPath: /csi
               name: socket-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml

@@ -7,9 +7,6 @@ subjects:
   - kind: ServiceAccount
     name: csi-attacher
     namespace: default
-  - kind: ServiceAccount
-    name: csi-node-sa
-    namespace: default
   - kind: ServiceAccount
     name: csi-provisioner
     namespace: default

test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml

@@ -1,12 +0,0 @@
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: psp-csi-mock-role
-subjects:
-  - kind: ServiceAccount
-    name: csi-driver-registrar
-    namespace: default
-roleRef:
-  kind: ClusterRole
-  name: e2e-test-privileged-psp
-  apiGroup: rbac.authorization.k8s.io