From df4c34e771cfd3bd2f09a718979aa8b12321916b Mon Sep 17 00:00:00 2001 From: Matthew Cary Date: Fri, 21 May 2021 17:43:36 +0000 Subject: [PATCH] Update pd csi driver images to use v1 images (CSINode, CSIDriver, etc) Change-Id: I51ddf0014dcfe476495e324ee33aa658e7199477 --- .../storage-csi/gce-pd/controller_ss.yaml | 83 ++++++++++++++++++- .../gce-pd/csi-controller-rbac.yaml | 36 +++++++- .../storage-csi/gce-pd/node_ds.yaml | 23 +++-- test/utils/image/manifest.go | 56 +++++++------ 4 files changed, 159 insertions(+), 39 deletions(-) diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml index 30209b5c5f0..64f00a2494b 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml @@ -21,28 +21,83 @@ spec: serviceAccountName: csi-gce-pd-controller-sa containers: - name: csi-snapshotter - image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2 + image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3 args: - "--v=5" - "--csi-address=/csi/csi.sock" + - "--metrics-address=:22014" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=300s" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace imagePullPolicy: Always volumeMounts: - name: socket-dir mountPath: /csi - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v1.6.0 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.1.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--feature-gates=Topology=true" + - "--http-endpoint=:22011" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=250s" + - "--extra-create-metadata" + # - "--run-controller-service=false" # disable the controller service of the CSI driver + # - "--run-node-service=false" # disable the node service of the CSI driver + - "--leader-election" + - "--default-fstype=ext4" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22011 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v2.2.0 + image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" + - "--http-endpoint=:22012" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=250s" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22012 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi @@ -51,12 +106,32 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" + - "--http-endpoint=:22013" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - "--handle-volume-inuse-error=false" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22013 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi - name: gce-pd-driver - image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v1.0.1-gke.0 + image: k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.2.2 args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml index 8ce1455ce0f..be74afaff21 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml @@ -69,6 +69,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] --- @@ -137,8 +140,8 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update", "patch"] - --- + kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -150,8 +153,37 @@ roleRef: kind: ClusterRole name: csi-gce-pd-snapshotter-role apiGroup: rbac.authorization.k8s.io - --- + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-leaderelection-role + namespace: gce-pd-csi-driver + labels: + k8s-app: gcp-compute-persistent-disk-csi-driver +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-leaderelection-binding + namespace: gce-pd-csi-driver + labels: + k8s-app: gcp-compute-persistent-disk-csi-driver +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa +roleRef: + kind: Role + name: csi-gce-pd-leaderelection-role + apiGroup: rbac.authorization.k8s.io +--- + # priviledged Pod Security Policy, previously defined via PrivilegedTestPSPClusterRoleBinding() kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml index 36a3c4d4f4f..b2b5b98a2c3 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml @@ -11,18 +11,14 @@ spec: labels: app: gcp-compute-persistent-disk-csi-driver spec: - # Host network must be used for interaction with Workload Identity in GKE - # since it replaces GCE Metadata Server with GKE Metadata Server. Remove - # this requirement when issue is resolved and before any exposure of - # metrics ports. - hostNetwork: true containers: - name: csi-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v1.3.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock" + - "--http-endpoint=:22013" lifecycle: preStop: exec: @@ -32,6 +28,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + ports: + - containerPort: 22013 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 volumeMounts: - name: plugin-dir mountPath: /csi @@ -40,10 +48,11 @@ spec: - name: gce-pd-driver securityContext: privileged: true - image: gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver:v0.7.0-gke.0 + image: k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.2.2 args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" + - "--run-controller-service=false" volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 654561c0368..4b64420035f 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -30,19 +30,20 @@ import ( // RegistryList holds public and private image registries type RegistryList struct { - GcAuthenticatedRegistry string `yaml:"gcAuthenticatedRegistry"` - E2eRegistry string `yaml:"e2eRegistry"` - PromoterE2eRegistry string `yaml:"promoterE2eRegistry"` - BuildImageRegistry string `yaml:"buildImageRegistry"` - InvalidRegistry string `yaml:"invalidRegistry"` - GcEtcdRegistry string `yaml:"gcEtcdRegistry"` - GcRegistry string `yaml:"gcRegistry"` - SigStorageRegistry string `yaml:"sigStorageRegistry"` - GcrReleaseRegistry string `yaml:"gcrReleaseRegistry"` - PrivateRegistry string `yaml:"privateRegistry"` - SampleRegistry string `yaml:"sampleRegistry"` - MicrosoftRegistry string `yaml:"microsoftRegistry"` - DockerLibraryRegistry string `yaml:"dockerLibraryRegistry"` + GcAuthenticatedRegistry string `yaml:"gcAuthenticatedRegistry"` + E2eRegistry string `yaml:"e2eRegistry"` + PromoterE2eRegistry string `yaml:"promoterE2eRegistry"` + BuildImageRegistry string `yaml:"buildImageRegistry"` + InvalidRegistry string `yaml:"invalidRegistry"` + GcEtcdRegistry string `yaml:"gcEtcdRegistry"` + GcRegistry string `yaml:"gcRegistry"` + SigStorageRegistry string `yaml:"sigStorageRegistry"` + GcrReleaseRegistry string `yaml:"gcrReleaseRegistry"` + PrivateRegistry string `yaml:"privateRegistry"` + SampleRegistry string `yaml:"sampleRegistry"` + MicrosoftRegistry string `yaml:"microsoftRegistry"` + DockerLibraryRegistry string `yaml:"dockerLibraryRegistry"` + CloudProviderGcpRegistry string `yaml:"cloudProviderGcpRegistry"` } // Config holds an images registry, name, and version @@ -89,19 +90,20 @@ func initReg() RegistryList { var ( initRegistry = RegistryList{ - GcAuthenticatedRegistry: "gcr.io/authenticated-image-pulling", - E2eRegistry: "gcr.io/kubernetes-e2e-test-images", - PromoterE2eRegistry: "k8s.gcr.io/e2e-test-images", - BuildImageRegistry: "k8s.gcr.io/build-image", - InvalidRegistry: "invalid.com/invalid", - GcEtcdRegistry: "k8s.gcr.io", - GcRegistry: "k8s.gcr.io", - SigStorageRegistry: "k8s.gcr.io/sig-storage", - PrivateRegistry: "gcr.io/k8s-authenticated-test", - SampleRegistry: "gcr.io/google-samples", - GcrReleaseRegistry: "gcr.io/gke-release", - MicrosoftRegistry: "mcr.microsoft.com", - DockerLibraryRegistry: "docker.io/library", + GcAuthenticatedRegistry: "gcr.io/authenticated-image-pulling", + E2eRegistry: "gcr.io/kubernetes-e2e-test-images", + PromoterE2eRegistry: "k8s.gcr.io/e2e-test-images", + BuildImageRegistry: "k8s.gcr.io/build-image", + InvalidRegistry: "invalid.com/invalid", + GcEtcdRegistry: "k8s.gcr.io", + GcRegistry: "k8s.gcr.io", + SigStorageRegistry: "k8s.gcr.io/sig-storage", + PrivateRegistry: "gcr.io/k8s-authenticated-test", + SampleRegistry: "gcr.io/google-samples", + GcrReleaseRegistry: "gcr.io/gke-release", + MicrosoftRegistry: "mcr.microsoft.com", + DockerLibraryRegistry: "docker.io/library", + CloudProviderGcpRegistry: "k8s.gcr.io/cloud-provider-gcp", } registry = initReg() @@ -405,6 +407,8 @@ func replaceRegistryInImageURLWithList(imageURL string, reg RegistryList) (strin registryAndUser = reg.GcAuthenticatedRegistry case initRegistry.DockerLibraryRegistry: registryAndUser = reg.DockerLibraryRegistry + case initRegistry.CloudProviderGcpRegistry: + registryAndUser = reg.CloudProviderGcpRegistry default: if countParts == 1 { // We assume we found an image from docker hub library