Merge pull request #100637 from pohly/hostpath-update

storage e2e: automate hostpath YAML updates, update sidecars but not driver

test/e2e/storage/drivers/csi.go

@@ -47,7 +47,9 @@ import (
 
 	"github.com/onsi/ginkgo"
 	"google.golang.org/grpc/codes"
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -142,7 +144,12 @@ func InitHostPathCSIDriver() storageframework.TestDriver {
 		storageframework.CapPVCDataSource:       true,
 		storageframework.CapControllerExpansion: true,
 		storageframework.CapSingleNodeVolume:    true,
-		storageframework.CapVolumeLimits:        true,
+
+		// This is needed for the
+		// testsuites/volumelimits.go `should support volume limits`
+		// test. --maxvolumespernode=10 gets
+		// added when patching the deployment.
+		storageframework.CapVolumeLimits: true,
 	}
 	return initHostPathCSIDriver("csi-hostpath",
 		capabilities,
@@ -152,7 +159,8 @@ func InitHostPathCSIDriver() storageframework.TestDriver {
 		},
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
-		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml",
@@ -220,16 +228,47 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo
 	}
 
 	o := utils.PatchCSIOptions{
-		OldDriverName:            h.driverInfo.Name,
+		OldDriverName:       h.driverInfo.Name,
-		NewDriverName:            config.GetUniqueDriverName(),
+		NewDriverName:       config.GetUniqueDriverName(),
-		DriverContainerName:      "hostpath",
+		DriverContainerName: "hostpath",
-		DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName()},
+		DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(),
+			// This is needed for the
+			// testsuites/volumelimits.go `should support volume limits`
+			// test.
+			"--maxvolumespernode=10",
+		},
 		ProvisionerContainerName: "csi-provisioner",
 		SnapshotterContainerName: "csi-snapshotter",
 		NodeName:                 node.Name,
 	}
 	cleanup, err := utils.CreateFromManifests(config.Framework, driverNamespace, func(item interface{}) error {
-		return utils.PatchCSIDeployment(config.Framework, o, item)
+		if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
+			return err
+		}
+
+		// Remove csi-external-health-monitor-agent and
+		// csi-external-health-monitor-controller
+		// containers. They are not needed for any of the
+		// tests and in practice apparently caused enough
+		// overhead that even unrelated tests timed out. For
+		// example, in the pull-kubernetes-e2e-kind test, 43
+		// out of 5771 tests failed, including tests from
+		// sig-node, sig-cli, sig-api-machinery, sig-network.
+		switch item := item.(type) {
+		case *appsv1.StatefulSet:
+			var containers []v1.Container
+			for _, container := range item.Spec.Template.Spec.Containers {
+				switch container.Name {
+				case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller":
+					// Remove these containers.
+				default:
+					// Keep the others.
+					containers = append(containers, container)
+				}
+			}
+			item.Spec.Template.Spec.Containers = containers
+		}
+		return nil
 	}, h.manifests...)
 
 	if err != nil {
@@ -408,7 +447,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver {
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
-		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml",
 	}
@@ -611,7 +650,25 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.P
 		FSGroupPolicy:     m.fsGroupPolicy,
 	}
 	cleanup, err := utils.CreateFromManifests(f, m.driverNamespace, func(item interface{}) error {
-		return utils.PatchCSIDeployment(f, o, item)
+		if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
+			return err
+		}
+
+		switch item := item.(type) {
+		case *rbacv1.ClusterRole:
+			if strings.HasPrefix(item.Name, "external-snapshotter-runner") {
+				// Re-enable access to secrets for the snapshotter sidecar for
+				// https://github.com/kubernetes/kubernetes/blob/6ede5ca95f78478fa627ecfea8136e0dff34436b/test/e2e/storage/csi_mock_volume.go#L1539-L1548
+				// It was disabled in https://github.com/kubernetes-csi/external-snapshotter/blob/501cc505846c03ee665355132f2da0ce7d5d747d/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml#L26-L32
+				item.Rules = append(item.Rules, rbacv1.PolicyRule{
+					APIGroups: []string{""},
+					Resources: []string{"secrets"},
+					Verbs:     []string{"get", "list"},
+				})
+			}
+		}
+
+		return nil
 	}, m.manifests...)
 
 	if err != nil {

test/e2e/testing-manifests/storage-csi/external-attacher/README.md

@@ -1 +0,0 @@
-The original file is https://github.com/kubernetes-csi/external-attacher/blob/VERSION/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-attacher/raw/v3.1.0/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI attacher.
 #
@@ -16,7 +20,7 @@ metadata:
   namespace: default
 
 ---
-# Attacher must be able to work with PVs, nodes and VolumeAttachments
+# Attacher must be able to work with PVs, CSINodes and VolumeAttachments
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -24,16 +28,13 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "patch"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments/status"]
     verbs: ["patch"]

test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml

@@ -0,0 +1,60 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-agent/rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor agent.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor agent, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-agent
+  # replace with non-default namespace name
+  namespace: default
+
+---
+# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-agent-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-agent-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-agent
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-agent-runner
+  apiGroup: rbac.authorization.k8s.io
+
+

test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml

@@ -0,0 +1,89 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-controller/rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor controller.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor controller, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-controller
+  # replace with non-default namespace name
+  namespace: default
+
+---
+# Health monitor controller must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Health monitor controller must be able to work with configmaps or leases in the current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # replace with non-default namespace name
+  namespace: default
+  name: external-health-monitor-controller-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role-cfg
+  # replace with non-default namespace name
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-health-monitor-controller-cfg
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/external-provisioner/README.md

@@ -1 +0,0 @@
-The original file is https://github.com/kubernetes-csi/external-provisioner/blob/VERSION/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-provisioner/raw/v2.1.1/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI provisioner.
 #
@@ -50,6 +54,13 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
+  # Access to volumeattachments is only needed when the CSI driver
+  # has the PUBLISH_UNPUBLISH_VOLUME controller capability.
+  # In that case, external-provisioner will watch volumeattachments
+  # to determine when it is safe to delete a volume.
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch"]
 
 ---
 kind: ClusterRoleBinding
@@ -84,6 +95,21 @@ rules:
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
   verbs: ["get", "watch", "list", "delete", "update", "create"]
+# Permissions for CSIStorageCapacity are only needed enabling the publishing
+# of storage capacity information.
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csistoragecapacities"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+# The GET permissions below are needed for walking up the ownership chain
+# for CSIStorageCapacity. They are sufficient for deployment via
+# StatefulSet (only needs to get Pod) and Deployment (needs to get
+# Pod and then ReplicaSet to find the Deployment).
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get"]
 
 ---
 kind: RoleBinding

test/e2e/testing-manifests/storage-csi/external-resizer/README.md

@@ -1 +0,0 @@
-The original file is https://github.com/kubernetes-csi/external-resizer/blob/VERSION/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-resizer/raw/v1.1.0/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI resizer.
 #
@@ -29,13 +33,16 @@ rules:
   #   verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "patch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
+    verbs: ["patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]

test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md

@@ -1 +0,0 @@
-The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/VERSION/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-snapshotter/raw/v4.0.0/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # Together with the RBAC file for external-provisioner, this YAML file
 # contains all RBAC objects that are necessary to run external CSI
 # snapshotter.
@@ -23,9 +27,13 @@ rules:
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
+  # Secret permission is optional.
-    resources: ["secrets"]
+  # Enable it if your driver needs secret.
-    verbs: ["get", "list"]
+  # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass.
+  # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details.
+  #  - apiGroups: [""]
+  #    resources: ["secrets"]
+  #    verbs: ["get", "list"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]

test/e2e/testing-manifests/storage-csi/hostpath/README.md

@@ -1,5 +1,4 @@
-A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
+The files in this directory are exact copys of "kubernetes-latest" in
-with some modifications:
+https://github.com/kubernetes-csi/csi-driver-host-path/tree/v1.6.2/deploy/
-- serviceAccountName is used instead of the deprecated serviceAccount
+
-- the RBAC roles from driver-registrar, external-attacher, external-provisioner
+Do not edit manually. Run test/e2e/testing-manifests/storage-csi/update-hostpath.sh to refresh the content.
-  and external-snapshotter are used

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml

@@ -1,17 +1,3 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-hostpath-attacher
-  labels:
-    app: csi-hostpath-attacher
-spec:
-  selector:
-    app: csi-hostpath-attacher
-  ports:
-    - name: dummy
-      port: 12345
-
----
 kind: StatefulSet
 apiVersion: apps/v1
 metadata:
@@ -40,7 +26,7 @@ spec:
       serviceAccountName: csi-attacher
       containers:
         - name: csi-attacher
-          image: k8s.gcr.io/sig-storage/csi-attacher:v2.2.0
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0
           args:
             - --v=5
             - --csi-address=/csi/csi.sock

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml

@@ -34,9 +34,39 @@ spec:
       labels:
         app: csi-hostpathplugin
     spec:
+      serviceAccount: csi-external-health-monitor-controller
       containers:
+        - name: csi-external-health-monitor-agent
+          image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-external-health-monitor-controller
+          image: k8s.gcr.io/sig-storage/csi-external-health-monitor-controller:v0.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: node-driver-registrar
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.3.0
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
           args:
             - --v=5
             - --csi-address=/csi/csi.sock
@@ -61,15 +91,15 @@ spec:
             name: csi-data-dir
 
         - name: hostpath
+          # WARNING: manually downgraded from 1.6.2 to 1.4.0 because 1.5.x and 1.6.x have
+          # a bug that causes E2E testing to fail (https://github.com/kubernetes-csi/csi-driver-host-path/pull/210#discussion_r605592438,
+          # https://github.com/kubernetes-csi/csi-driver-host-path/issues/251).
           image: k8s.gcr.io/sig-storage/hostpathplugin:v1.4.0
           args:
             - "--drivername=hostpath.csi.k8s.io"
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(KUBE_NODE_NAME)"
-            # The only difference to github.com/kubernetes-csi/csi-driver-host-path/deploy
-            # - we have a tests that checks node limits.
-            - "--maxvolumespernode=10"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
@@ -109,7 +139,7 @@ spec:
           volumeMounts:
           - mountPath: /csi
             name: socket-dir
-          image: k8s.gcr.io/sig-storage/livenessprobe:v1.1.0
+          image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0
           args:
           - --csi-address=/csi/csi.sock
           - --health-port=9898

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml

@@ -1,17 +1,3 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-hostpath-provisioner 
-  labels:
-    app: csi-hostpath-provisioner 
-spec:
-  selector:
-    app: csi-hostpath-provisioner 
-  ports:
-    - name: dummy
-      port: 12345
-
----
 kind: StatefulSet
 apiVersion: apps/v1
 metadata:
@@ -40,7 +26,7 @@ spec:
       serviceAccountName: csi-provisioner
       containers:
         - name: csi-provisioner
-          image: k8s.gcr.io/sig-storage/csi-provisioner:v1.6.0
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.1.1
           args:
             - -v=5
             - --csi-address=/csi/csi.sock

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml

@@ -1,17 +1,3 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-hostpath-resizer
-  labels:
-    app: csi-hostpath-resizer
-spec:
-  selector:
-    app: csi-hostpath-resizer
-  ports:
-    - name: dummy
-      port: 12345
-
----
 kind: StatefulSet
 apiVersion: apps/v1
 metadata:
@@ -40,7 +26,7 @@ spec:
       serviceAccountName: csi-resizer
       containers:
         - name: csi-resizer
-          image: k8s.gcr.io/sig-storage/csi-resizer:v0.5.0
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
           args:
             - -v=5
             - -csi-address=/csi/csi.sock

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml

@@ -1,17 +1,3 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: csi-hostpath-snapshotter
-  labels:
-    app: csi-hostpath-snapshotter
-spec:
-  selector:
-    app: csi-hostpath-snapshotter
-  ports:
-    - name: dummy
-      port: 12345
-
----
 kind: StatefulSet
 apiVersion: apps/v1
 metadata:
@@ -40,7 +26,7 @@ spec:
       serviceAccount: csi-snapshotter
       containers:
         - name: csi-snapshotter
-          image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
           args:
             - -v=5
             - --csi-address=/csi/csi.sock

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml

@@ -0,0 +1,64 @@
+# WARNING: this is only for testing purposes. Do not install in a production
+# cluster.
+#
+# This exposes the hostpath's Unix domain csi.sock as a TCP port to the
+# outside world. The mapping from Unix domain socket to TCP is done
+# by socat.
+#
+# This is useful for testing with csi-sanity or csc.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: hostpath-service
+spec:
+  type: NodePort
+  selector:
+    app: csi-hostpath-socat
+  ports:
+  - port: 10000 # fixed port inside the pod, dynamically allocated port outside
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-socat
+spec:
+  serviceName: "csi-hostpath-socat"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-socat
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-socat
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - csi-hostpathplugin
+            topologyKey: kubernetes.io/hostname
+      containers:
+        - name: socat
+          image: alpine/socat:1.0.3
+          args:
+            - tcp-listen:10000,fork,reuseaddr
+            - unix-connect:/csi/csi.sock
+          securityContext:
+            # This is necessary only for systems with SELinux, where
+            # non-privileged sidecar containers cannot access unix domain socket
+            # created by privileged CSI driver container.
+            privileged: true
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml

@@ -16,6 +16,9 @@ subjects:
   - kind: ServiceAccount
     name: csi-resizer
     namespace: default
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: e2e-test-privileged-psp

test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml

@@ -1,7 +0,0 @@
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: csi-hostpath-sc
-provisioner: csi-hostpath
-reclaimPolicy: Delete
-volumeBindingMode: Immediate

test/e2e/testing-manifests/storage-csi/update-hostpath.sh

@@ -0,0 +1,128 @@
+#!/bin/sh
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script will update all sidecar RBAC files and the CSI hostpath
+# deployment files such that they match what is in a hostpath driver
+# release.
+#
+# Beware that this will wipe out all local modifications!
+
+# Can be a tag or a branch.
+script="$0"
+hostpath_version="$1"
+
+if ! [ "$hostpath_version" ]; then
+    cat >&2 <<EOF
+Usage: $0 <hostpath tag or branch name>
+
+Required parameter is missing.
+EOF
+    exit 1
+fi
+
+set -xe
+cd "$(dirname "$0")"
+
+# Remove stale files.
+rm -rf external-attacher external-provisioner external-resizer external-snapshotter external-health-monitor hostpath csi-driver-host-path
+
+# Check out desired release.
+git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
+(cd csi-driver-host-path && git checkout "$hostpath_version")
+trap "rm -rf csi-driver-host-path" EXIT
+
+# Main YAML files.
+mkdir hostpath
+cat >hostpath/README.md <<EOF
+The files in this directory are exact copys of "kubernetes-latest" in
+https://github.com/kubernetes-csi/csi-driver-host-path/tree/$hostpath_version/deploy/
+
+Do not edit manually. Run $script to refresh the content.
+EOF
+cp -r csi-driver-host-path/deploy/kubernetes-latest/hostpath hostpath/
+cat >hostpath/hostpath/e2e-test-rbac.yaml <<EOF
+# priviledged Pod Security Policy, previously defined just for gcePD via PrivilegedTestPSPClusterRoleBinding()
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp-csi-hostpath-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-provisioner
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-snapshotter
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-resizer
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: e2e-test-privileged-psp
+  apiGroup: rbac.authorization.k8s.io
+EOF
+
+download () {
+    project="$1"
+    path="$2"
+    tag="$3"
+    rbac="$4"
+
+    mkdir -p "$project/$path"
+    url="https://github.com/kubernetes-csi/$project/raw/$tag/deploy/kubernetes/$path/$rbac"
+    cat >"$project/$path/$rbac" <<EOF
+# Do not edit, downloaded from $url
+# for csi-driver-host-path $hostpath_version
+# by $script
+#
+EOF
+    curl --fail --location "$url" >>"$project/$path/$rbac"
+}
+
+# RBAC files for each sidecar.
+# This relies on the convention that "external-something" has "csi-something" as image name.
+# external-health-monitor is special, it has two images.
+# The repository for each image is ignored.
+images=$(grep -r '^ *image:.*csi' hostpath/hostpath | sed -e 's;.*image:.*/;;' | grep -v 'node-driver-registrar' | sort -u)
+for image in $images; do
+    tag=$(echo "$image" | sed -e 's/.*://')
+    path=
+    rbac="rbac.yaml"
+    case $image in
+        csi-external-*)
+            # csi-external-health-monitor-agent:v0.2.0
+            project=$(echo "$image" | sed -e 's/csi-\(.*\)-[^:]*:.*/\1/')
+            path=$(echo "$image" | sed -e 's/csi-\([^:]*\):.*/\1/')
+            ;;
+        *)
+            project=$(echo "$image" | sed -e 's/:.*//' -e 's/^csi/external/')
+            case $project in
+                external-snapshotter)
+                    # Another special case...
+                    path="csi-snapshotter"
+                    rbac="rbac-csi-snapshotter.yaml"
+                    ;;
+            esac
+            ;;
+    esac
+    download "$project" "$path" "$tag" "$rbac"
+done