diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 2035c8d6a11..c5775767f36 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -47,7 +47,9 @@ import ( "github.com/onsi/ginkgo" "google.golang.org/grpc/codes" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -142,7 +144,12 @@ func InitHostPathCSIDriver() storageframework.TestDriver { storageframework.CapPVCDataSource: true, storageframework.CapControllerExpansion: true, storageframework.CapSingleNodeVolume: true, - storageframework.CapVolumeLimits: true, + + // This is needed for the + // testsuites/volumelimits.go `should support volume limits` + // test. --maxvolumespernode=10 gets + // added when patching the deployment. + storageframework.CapVolumeLimits: true, } return initHostPathCSIDriver("csi-hostpath", capabilities, @@ -152,7 +159,8 @@ func InitHostPathCSIDriver() storageframework.TestDriver { }, "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml", + "test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml", "test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml", @@ -220,16 +228,47 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo } o := utils.PatchCSIOptions{ - OldDriverName: h.driverInfo.Name, - NewDriverName: config.GetUniqueDriverName(), - DriverContainerName: "hostpath", - DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName()}, + OldDriverName: h.driverInfo.Name, + NewDriverName: config.GetUniqueDriverName(), + DriverContainerName: "hostpath", + DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(), + // This is needed for the + // testsuites/volumelimits.go `should support volume limits` + // test. + "--maxvolumespernode=10", + }, ProvisionerContainerName: "csi-provisioner", SnapshotterContainerName: "csi-snapshotter", NodeName: node.Name, } cleanup, err := utils.CreateFromManifests(config.Framework, driverNamespace, func(item interface{}) error { - return utils.PatchCSIDeployment(config.Framework, o, item) + if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { + return err + } + + // Remove csi-external-health-monitor-agent and + // csi-external-health-monitor-controller + // containers. They are not needed for any of the + // tests and in practice apparently caused enough + // overhead that even unrelated tests timed out. For + // example, in the pull-kubernetes-e2e-kind test, 43 + // out of 5771 tests failed, including tests from + // sig-node, sig-cli, sig-api-machinery, sig-network. + switch item := item.(type) { + case *appsv1.StatefulSet: + var containers []v1.Container + for _, container := range item.Spec.Template.Spec.Containers { + switch container.Name { + case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller": + // Remove these containers. + default: + // Keep the others. + containers = append(containers, container) + } + } + item.Spec.Template.Spec.Containers = containers + } + return nil }, h.manifests...) if err != nil { @@ -408,7 +447,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver { "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", } @@ -611,7 +650,25 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.P FSGroupPolicy: m.fsGroupPolicy, } cleanup, err := utils.CreateFromManifests(f, m.driverNamespace, func(item interface{}) error { - return utils.PatchCSIDeployment(f, o, item) + if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { + return err + } + + switch item := item.(type) { + case *rbacv1.ClusterRole: + if strings.HasPrefix(item.Name, "external-snapshotter-runner") { + // Re-enable access to secrets for the snapshotter sidecar for + // https://github.com/kubernetes/kubernetes/blob/6ede5ca95f78478fa627ecfea8136e0dff34436b/test/e2e/storage/csi_mock_volume.go#L1539-L1548 + // It was disabled in https://github.com/kubernetes-csi/external-snapshotter/blob/501cc505846c03ee665355132f2da0ce7d5d747d/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml#L26-L32 + item.Rules = append(item.Rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list"}, + }) + } + } + + return nil }, m.manifests...) if err != nil { diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md b/test/e2e/testing-manifests/storage-csi/external-attacher/README.md deleted file mode 100644 index a8766137cd0..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-attacher/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml index 4d6dd506ce1..9111dc30c77 100644 --- a/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-attacher/raw/v3.1.0/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI attacher. # @@ -16,7 +20,7 @@ metadata: namespace: default --- -# Attacher must be able to work with PVs, nodes and VolumeAttachments +# Attacher must be able to work with PVs, CSINodes and VolumeAttachments kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -24,16 +28,13 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] diff --git a/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml new file mode 100644 index 00000000000..f698e8fdaff --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml @@ -0,0 +1,60 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-agent/rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# +# This YAML file contains all RBAC objects that are necessary to run external +# CSI health monitor agent. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# health monitor agent, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-external-health-monitor-agent + # replace with non-default namespace name + namespace: default + +--- +# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-health-monitor-agent-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-agent-role +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-agent + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-health-monitor-agent-runner + apiGroup: rbac.authorization.k8s.io + + diff --git a/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml new file mode 100644 index 00000000000..1ec35db8cf5 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml @@ -0,0 +1,89 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-controller/rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# +# This YAML file contains all RBAC objects that are necessary to run external +# CSI health monitor controller. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - decide whether the deployment replicates the external CSI +# health monitor controller, in which case leadership election must be enabled; +# this influences the RBAC setup, see below + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default + +--- +# Health monitor controller must be able to work with PVs, PVCs, Nodes and Pods +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-health-monitor-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-controller-role +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: external-health-monitor-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +# Health monitor controller must be able to work with configmaps or leases in the current namespace +# if (and only if) leadership election is enabled +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: external-health-monitor-controller-cfg +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-external-health-monitor-controller-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: csi-external-health-monitor-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: external-health-monitor-controller-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md b/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md deleted file mode 100644 index 982a1adae98..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-provisioner/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml index 35b68801595..68cc4c1dfac 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-provisioner/raw/v2.1.1/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI provisioner. # @@ -50,6 +54,13 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -84,6 +95,21 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +# Permissions for CSIStorageCapacity are only needed enabling the publishing +# of storage capacity information. +- apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# The GET permissions below are needed for walking up the ownership chain +# for CSIStorageCapacity. They are sufficient for deployment via +# StatefulSet (only needs to get Pod) and Deployment (needs to get +# Pod and then ReplicaSet to find the Deployment). +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +- apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] --- kind: RoleBinding diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md b/test/e2e/testing-manifests/storage-csi/external-resizer/README.md deleted file mode 100644 index 4e7e248a7ff..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-resizer/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml index 17ed01f8381..590c5420836 100644 --- a/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-resizer/raw/v1.1.0/deploy/kubernetes//rbac.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # This YAML file contains all RBAC objects that are necessary to run external # CSI resizer. # @@ -29,13 +33,16 @@ rules: # verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] + verbs: ["patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md b/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md deleted file mode 100644 index d96fc148794..00000000000 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is https://github.com/kubernetes-csi/external-snapshotter/blob/VERSION/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml similarity index 79% rename from test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml rename to test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml index ca4a54b6e05..6577637c773 100644 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml @@ -1,3 +1,7 @@ +# Do not edit, downloaded from https://github.com/kubernetes-csi/external-snapshotter/raw/v4.0.0/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml +# for csi-driver-host-path v1.6.2 +# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh +# # Together with the RBAC file for external-provisioner, this YAML file # contains all RBAC objects that are necessary to run external CSI # snapshotter. @@ -23,9 +27,13 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/README.md b/test/e2e/testing-manifests/storage-csi/hostpath/README.md index c2990622700..2f06c7e550a 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/README.md +++ b/test/e2e/testing-manifests/storage-csi/hostpath/README.md @@ -1,5 +1,4 @@ -A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example, -with some modifications: -- serviceAccountName is used instead of the deprecated serviceAccount -- the RBAC roles from driver-registrar, external-attacher, external-provisioner - and external-snapshotter are used +The files in this directory are exact copys of "kubernetes-latest" in +https://github.com/kubernetes-csi/csi-driver-host-path/tree/v1.6.2/deploy/ + +Do not edit manually. Run test/e2e/testing-manifests/storage-csi/update-hostpath.sh to refresh the content. diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml index 2c23f75c71c..6c5a14a146c 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-attacher - labels: - app: csi-hostpath-attacher -spec: - selector: - app: csi-hostpath-attacher - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-attacher containers: - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v2.2.0 + image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0 args: - --v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml index 5997d00e71d..5744ea6186e 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml @@ -34,9 +34,39 @@ spec: labels: app: csi-hostpathplugin spec: + serviceAccount: csi-external-health-monitor-controller containers: + - name: csi-external-health-monitor-agent + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-external-health-monitor-controller + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-controller:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: node-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.3.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 args: - --v=5 - --csi-address=/csi/csi.sock @@ -61,15 +91,15 @@ spec: name: csi-data-dir - name: hostpath + # WARNING: manually downgraded from 1.6.2 to 1.4.0 because 1.5.x and 1.6.x have + # a bug that causes E2E testing to fail (https://github.com/kubernetes-csi/csi-driver-host-path/pull/210#discussion_r605592438, + # https://github.com/kubernetes-csi/csi-driver-host-path/issues/251). image: k8s.gcr.io/sig-storage/hostpathplugin:v1.4.0 args: - "--drivername=hostpath.csi.k8s.io" - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - # The only difference to github.com/kubernetes-csi/csi-driver-host-path/deploy - # - we have a tests that checks node limits. - - "--maxvolumespernode=10" env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock @@ -109,7 +139,7 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - image: k8s.gcr.io/sig-storage/livenessprobe:v1.1.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 args: - --csi-address=/csi/csi.sock - --health-port=9898 diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml index 856dbff120c..0cbc9910cea 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-provisioner - labels: - app: csi-hostpath-provisioner -spec: - selector: - app: csi-hostpath-provisioner - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-provisioner containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v1.6.0 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.1.1 args: - -v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml index 0294f8b2a38..524f9ed4f36 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-resizer - labels: - app: csi-hostpath-resizer -spec: - selector: - app: csi-hostpath-resizer - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccountName: csi-resizer containers: - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v0.5.0 + image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 args: - -v=5 - -csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml index 66c3882ff64..3a61d9e14e4 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml @@ -1,17 +1,3 @@ -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-snapshotter - labels: - app: csi-hostpath-snapshotter -spec: - selector: - app: csi-hostpath-snapshotter - ports: - - name: dummy - port: 12345 - ---- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -40,7 +26,7 @@ spec: serviceAccount: csi-snapshotter containers: - name: csi-snapshotter - image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2 + image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0 args: - -v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 00000000000..188a5bde870 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,64 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service +spec: + type: NodePort + selector: + app: csi-hostpath-socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-socat + template: + metadata: + labels: + app: csi-hostpath-socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml index dde7ce78258..3cd01df1f74 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml @@ -16,6 +16,9 @@ subjects: - kind: ServiceAccount name: csi-resizer namespace: default + - kind: ServiceAccount + name: csi-external-health-monitor-controller + namespace: default roleRef: kind: ClusterRole name: e2e-test-privileged-psp diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml deleted file mode 100644 index c92797167e6..00000000000 --- a/test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-hostpath-sc -provisioner: csi-hostpath -reclaimPolicy: Delete -volumeBindingMode: Immediate diff --git a/test/e2e/testing-manifests/storage-csi/update-hostpath.sh b/test/e2e/testing-manifests/storage-csi/update-hostpath.sh new file mode 100755 index 00000000000..493baedaa25 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/update-hostpath.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will update all sidecar RBAC files and the CSI hostpath +# deployment files such that they match what is in a hostpath driver +# release. +# +# Beware that this will wipe out all local modifications! + +# Can be a tag or a branch. +script="$0" +hostpath_version="$1" + +if ! [ "$hostpath_version" ]; then + cat >&2 < + +Required parameter is missing. +EOF + exit 1 +fi + +set -xe +cd "$(dirname "$0")" + +# Remove stale files. +rm -rf external-attacher external-provisioner external-resizer external-snapshotter external-health-monitor hostpath csi-driver-host-path + +# Check out desired release. +git clone https://github.com/kubernetes-csi/csi-driver-host-path.git +(cd csi-driver-host-path && git checkout "$hostpath_version") +trap "rm -rf csi-driver-host-path" EXIT + +# Main YAML files. +mkdir hostpath +cat >hostpath/README.md <hostpath/hostpath/e2e-test-rbac.yaml <"$project/$path/$rbac" <>"$project/$path/$rbac" +} + +# RBAC files for each sidecar. +# This relies on the convention that "external-something" has "csi-something" as image name. +# external-health-monitor is special, it has two images. +# The repository for each image is ignored. +images=$(grep -r '^ *image:.*csi' hostpath/hostpath | sed -e 's;.*image:.*/;;' | grep -v 'node-driver-registrar' | sort -u) +for image in $images; do + tag=$(echo "$image" | sed -e 's/.*://') + path= + rbac="rbac.yaml" + case $image in + csi-external-*) + # csi-external-health-monitor-agent:v0.2.0 + project=$(echo "$image" | sed -e 's/csi-\(.*\)-[^:]*:.*/\1/') + path=$(echo "$image" | sed -e 's/csi-\([^:]*\):.*/\1/') + ;; + *) + project=$(echo "$image" | sed -e 's/:.*//' -e 's/^csi/external/') + case $project in + external-snapshotter) + # Another special case... + path="csi-snapshotter" + rbac="rbac-csi-snapshotter.yaml" + ;; + esac + ;; + esac + download "$project" "$path" "$tag" "$rbac" +done