Upgrade dependencies: olslim version, sidecars, deployment

Dockerfile

@@ -5,7 +5,7 @@
 # This is the Dockerfile for Oracle ZFS Storage Appliance CSI Driver
 #
 
-FROM container-registry.oracle.com/os/oraclelinux:7-slim
+FROM container-registry.oracle.com/os/oraclelinux:9-slim
 LABEL maintainers="Oracle"
 LABEL description="Oracle ZFS Storage Appliance CSI Driver for Kubernetes"
 
@@ -14,7 +14,7 @@ ENV http_proxy=$var_proxy
 ENV https_proxy=$var_proxy
 
 # Add util-linux to get a new version of losetup.
-RUN yum -y install iscsi-initiator-utils nfs-utils e2fsprogs xfsprogs && yum clean all
+RUN microdnf -y install iscsi-initiator-utils nfs-utils e2fsprogs xfsprogs && microdnf clean all
 
 ENV http_proxy ""
 ENV https_proxy ""

INSTALLATION.md

@@ -51,21 +51,15 @@ Ensure the following information and requirements can be met prior to installati
     Make sure you have access to the registry or registries containing these images from the worker nodes. The image pull
     policy (`imagePullPolicy`) is set to `IfNotPresent` in the deployment files. During the first deployment the
     Container Runtime will likely try to pull them. If your Container Runtime cannot access the images you will have to
-    pull them manually before deployment. The required images are: 
+    pull them manually before deployment.
 
-    * node-driver-registar v2.0.0+.
+    For this driver, the [Oracle Container Registry](https://container-registry.oracle.com/) will be used.
-    * external-attacher v3.0.2+.
-    * external-provisioner v2.0.5+.
-    * external-resizer v1.1.0+.
-    * external-snapshotter v3.0.3+.
-
-    The common container images for those images are:
     
-    * k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.0  
+    * container-registry.oracle.com/olcne_developer/csi-node-driver-registrar:v2.7.0  
-    * k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
+    * container-registry.oracle.com/olcne_developer/csi-attacher:v4.1.0
-    * k8s.gcr.io/sig-storage/csi-provisioner:v2.0.5 
+    * container-registry.oracle.com/olcne_developer/csi-provisioner:v3.4.0 
-    * k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 
+    * container-registry.oracle.com/olcne_developer/csi-resizer:v1.7.0
-    * k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3 
+    * container-registry.oracle.com/olcne_developer/csi-snapshotter:v6.2.1
     
 * Plugin image
 

README.md

@@ -4,12 +4,13 @@ This plugin supports Oracle ZFS Storage Appliance
 as a backend for block storage (iSCSI volumes) and file storage (NFS).
 
 | CSI Plugin Version | Supported CSI Versions | Supported Kubernetes Versions | Persistence | Supported Access Modes | Dynamic Provisioning | Raw Block Support |
-| --- | --- | --- | --- | --- | --- | --- |
+|--------------------|------------------------|-------------------------------| --- | --- | --- | --- |
-| v1.1.0 | v1.0+ | v1.20.X+ | Persistent | Read/Write Once (for Block), ReadWriteMany (for File) | Yes | Yes |
+| v1.1.0             | v1.0+                  | v1.20.X+                      | Persistent | Read/Write Once (for Block), ReadWriteMany (for File) | Yes | Yes |
+| v1.8.0             | v1.8.0+                | v1.26.X+                      | Persistent | Read/Write Once (for Block), ReadWriteMany (for File) | Yes | Yes |
 
 ## Requirements
 
-* Kubernetes v1.20 or above (Oracle Linux Cloud Native Environment 1.3)
+* Kubernetes v1.26 or above (Oracle Linux Cloud Native Environment 1.3)
 * A Container runtime implementing the Kubernetes Container Runtime Interface (ex. CRI-O)
 * An Oracle ZFS Storage Appliance running Appliance Kit Version 8.8 or above. This plugin may work with previous 
   versions but it is not tested with them. It is possible to use this 

VERSION

@@ -1 +1 @@
-1.1.0
+1.8.0

deploy/helm/k8s-1.27/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+name: zfssa-csi
+version: 1.8.0
+description: Deploys Oracle ZFS Storage Appliance CSI Plugin.

deploy/helm/k8s-1.27/templates/00-zfssa-csi-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+stringData:
+  zfssa.yaml: |
+    username: {{ .Values.zfssaInformation.username }}
+    password: {{ .Values.zfssaInformation.password }}
+kind: Secret
+metadata:
+  name: oracle.zfssa.csi.node
+  namespace: {{ .Values.deployment.namespace }}

deploy/helm/k8s-1.27/templates/01-zfssa-csi-certs.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  zfssa.crt: {{ .Values.zfssaInformation.cert }}
+kind: Secret
+metadata:
+  name: oracle.zfssa.csi.node.certs
+type: Opaque
+

deploy/helm/k8s-1.27/templates/02-zfssa-csi-rbac.yaml

@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: zfssa-csi
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: zfssa-csi-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "create", "delete", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch", "delete", "get"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents/status" ]
+    verbs: [ "update" ]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csidrivers"]
+    verbs: ["create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "create", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: zfssa-csi-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: zfssa-csi
+    namespace: {{ .Values.deployment.namespace }}
+roleRef:
+  kind: ClusterRole
+  name: zfssa-csi-role
+  apiGroup: rbac.authorization.k8s.io

deploy/helm/k8s-1.27/templates/03-zfssa-csi-driver.yaml

@@ -0,0 +1,139 @@
+# Service defined here, plus serviceName below in StatefulSet,
+# are needed only because of condition explained in
+# https://github.com/kubernetes/kubernetes/issues/69608
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: zfssa-csi-driver
+  namespace: {{ .Values.deployment.namespace }}
+spec:
+  attachRequired: true
+  podInfoOnMount: true
+  volumeLifecycleModes:
+    - Persistent
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: zfssa-csi-nodeplugin
+  namespace: {{ .Values.deployment.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: zfssa-csi-nodeplugin
+  template:
+    metadata:
+      labels:
+        app: zfssa-csi-nodeplugin
+    spec:
+      serviceAccount: zfssa-csi
+      hostNetwork: true
+      containers:
+        - name: node-driver-registrar
+          image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiNodeDriverRegistrar.name }}:{{ .Values.images.csiNodeDriverRegistrar.tag }}
+          args:
+            - --v=5
+            - --csi-address=/plugin/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/com.oracle.zfssabs/csi.sock
+          imagePullPolicy:  {{ .Values.image.pullPolicy }}
+          securityContext:
+            # This is necessary only for systems with SELinux, where
+            # non-privileged sidecar containers cannot access unix domain socket
+            # created by privileged CSI driver container.
+            privileged: true
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: {{ .Values.paths.pluginDir.mountPath }}
+            - name: registration-dir
+              mountPath: /registration
+
+        - name: zfssabs
+          image: {{ .Values.image.zfssaBase }}{{ .Values.images.zfssaCsiDriver.name }}:{{ .Values.images.zfssaCsiDriver.tag }}
+          args:
+            - "--drivername=zfssa-csi-driver.oracle.com"
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://plugin/csi.sock
+            - name: LOG_LEVEL
+              value: "5"
+            - name: ZFSSA_TARGET
+              value: {{ .Values.zfssaInformation.target }}
+            - name: ZFSSA_INSECURE
+              value: "False"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          imagePullPolicy:  {{ .Values.image.pullPolicy }}
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: {{ .Values.paths.pluginDir.mountPath }}
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+            - name: plugins-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: Bidirectional
+            - name: dev-dir
+              mountPath: /dev
+            - name: zfssa-credentials
+              mountPath: "/mnt/zfssa"
+              readOnly: true
+            - name: certs
+              mountPath: "/mnt/certs"
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: {{ .Values.paths.pluginDir.hostPath }}
+            type: DirectoryOrCreate
+        - name: mountpoint-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+        - name: plugins-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+        - name: dev-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: zfssa-credentials
+          secret:
+            secretName: oracle.zfssa.csi.node
+            items:
+              - key: zfssa.yaml
+                path: zfssa.yaml
+        - name: certs
+          secret:
+            secretName: oracle.zfssa.csi.node.certs
+            items:
+              - key: zfssa.crt
+                path: zfssa.crt

deploy/helm/k8s-1.27/templates/04-zfssa-csi-provisioner.yaml

@@ -0,0 +1,90 @@
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: zfssa-csi-provisioner
+  namespace: {{ .Values.deployment.namespace }}
+spec:
+  serviceName: "zfssa-csi-provisioner"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: zfssa-csi-provisioner
+  template:
+    metadata:
+      labels:
+        app: zfssa-csi-provisioner
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - zfssa-csi-nodeplugin
+              topologyKey: kubernetes.io/hostname
+      serviceAccountName: zfssa-csi
+      containers:
+        - name: zfssa-csi-snapshotter
+          image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiSnapshotter.name }}:{{ .Values.images.csiSnapshotter.tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=false"
+          env:
+            - name: ADDRESS
+              value: /plugin/csi.sock
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /plugin
+        - name: zfssa-csi-resizer
+          image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiResizer.name }}:{{ .Values.images.csiResizer.tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /plugin/csi.sock
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /plugin
+        - name: zfssa-csi-provisioner
+          image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiProvisioner.name }}:{{ .Values.images.csiProvisioner.tag }}
+          args:
+            - -v=5
+            - --csi-address=/plugin/csi.sock
+            - --timeout=30s
+            - --feature-gates=Topology=true
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            # This is necessary only for systems with SELinux, where
+            # non-privileged sidecar containers cannot access unix domain socket
+            # created by privileged CSI driver container.
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /plugin
+        - name: zfssa-csi-attacher
+          image: {{ .Values.image.sidecarBase }}{{ .Values.images.csiAttacher.name }}:{{ .Values.images.csiAttacher.tag }}
+          args:
+            - --v=5
+            - --csi-address=/plugin/csi.sock
+            # securityContext:
+            # This is necessary only for systems with SELinux, where
+            # non-privileged sidecar containers cannot access unix domain socket
+            # created by privileged CSI driver container.
+            # privileged: true
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: {{ .Values.paths.pluginDir.mountPath }}
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: {{ .Values.paths.pluginDir.hostPath }}
+            type: DirectoryOrCreate

deploy/helm/k8s-1.27/values.yaml

@@ -0,0 +1,42 @@
+# Global docker image setting
+image:
+  sidecarBase: container-registry.oracle.com/olcne_developer/
+  zfssaBase: iad.ocir.io/zs/store/csi/
+  pullPolicy: Always
+
+# Define all the images that will be used during helm chart deployment
+images:
+  csiNodeDriverRegistrar:
+    name: csi-node-driver-registrar
+    tag: "v2.7.0"
+  zfssaCsiDriver:
+    name: zfssa-csi-driver
+    tag: "v1.8.0"
+  csiProvisioner:
+    name: csi-provisioner
+    tag: "v3.4.0"
+  csiAttacher:
+    name: csi-attacher
+    tag: "v4.1.0"
+  csiResizer:
+    name: csi-resizer
+    tag: "v1.7.0"
+  csiSnapshotter:
+    name: csi-snapshotter
+    tag: "v6.2.1"
+
+paths:
+  pluginDir:
+    hostPath: "/var/lib/kubelet/plugins/com.oracle.zfssabs"
+    mountPath: "/plugin"
+
+deployment:
+  namespace: default
+
+# ZFSSA-specific information
+# It is desirable to provision a normal login user with required authorizations.
+zfssaInformation:
+  username: text-string
+  password: text-string
+  target: text-string
+  cert: cert-base64-encoded

deploy/k8s-1.27/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml

@@ -0,0 +1,85 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .driver
+    name: Driver
+    type: string
+  - JSONPath: .deletionPolicy
+    description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass
+      should be deleted when its bound VolumeSnapshot is deleted.
+    name: DeletionPolicy
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    singular: volumesnapshotclass
+  preserveUnknownFields: false
+  scope: Cluster
+  subresources: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshotClass specifies parameters that a underlying storage
+        system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+        is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+        are non-namespaced
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        deletionPolicy:
+          description: deletionPolicy determines whether a VolumeSnapshotContent created
+            through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
+            is deleted. Supported values are "Retain" and "Delete". "Retain" means
+            that the VolumeSnapshotContent and its physical snapshot on underlying
+            storage system are kept. "Delete" means that the VolumeSnapshotContent
+            and its physical snapshot on underlying storage system are deleted. Required.
+          enum:
+          - Delete
+          - Retain
+          type: string
+        driver:
+          description: driver is the name of the storage driver that handles this
+            VolumeSnapshotClass. Required.
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        parameters:
+          additionalProperties:
+            type: string
+          description: parameters is a key-value map with storage driver specific
+            parameters for creating snapshots. These values are opaque to Kubernetes.
+          type: object
+      required:
+      - deletionPolicy
+      - driver
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

deploy/k8s-1.27/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml

@@ -0,0 +1,233 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.readyToUse
+    description: Indicates if a snapshot is ready to be used to restore a volume.
+    name: ReadyToUse
+    type: boolean
+  - JSONPath: .status.restoreSize
+    description: Represents the complete size of the snapshot in bytes
+    name: RestoreSize
+    type: integer
+  - JSONPath: .spec.deletionPolicy
+    description: Determines whether this VolumeSnapshotContent and its physical snapshot
+      on the underlying storage system should be deleted when its bound VolumeSnapshot
+      is deleted.
+    name: DeletionPolicy
+    type: string
+  - JSONPath: .spec.driver
+    description: Name of the CSI driver used to create the physical snapshot on the
+      underlying storage system.
+    name: Driver
+    type: string
+  - JSONPath: .spec.volumeSnapshotClassName
+    description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+    name: VolumeSnapshotClass
+    type: string
+  - JSONPath: .spec.volumeSnapshotRef.name
+    description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+      object is bound.
+    name: VolumeSnapshot
+    type: string
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    singular: volumesnapshotcontent
+  preserveUnknownFields: false
+  scope: Cluster
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+        object in the underlying storage system
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: spec defines properties of a VolumeSnapshotContent created
+            by the underlying storage system. Required.
+          properties:
+            deletionPolicy:
+              description: deletionPolicy determines whether this VolumeSnapshotContent
+                and its physical snapshot on the underlying storage system should
+                be deleted when its bound VolumeSnapshot is deleted. Supported values
+                are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                and its physical snapshot on underlying storage system are kept. "Delete"
+                means that the VolumeSnapshotContent and its physical snapshot on
+                underlying storage system are deleted. In dynamic snapshot creation
+                case, this field will be filled in with the "DeletionPolicy" field
+                defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For
+                pre-existing snapshots, users MUST specify this field when creating
+                the VolumeSnapshotContent object. Required.
+              enum:
+              - Delete
+              - Retain
+              type: string
+            driver:
+              description: driver is the name of the CSI driver used to create the
+                physical snapshot on the underlying storage system. This MUST be the
+                same as the name returned by the CSI GetPluginName() call for that
+                driver. Required.
+              type: string
+            source:
+              description: source specifies from where a snapshot will be created.
+                This field is immutable after creation. Required.
+              properties:
+                snapshotHandle:
+                  description: snapshotHandle specifies the CSI "snapshot_id" of a
+                    pre-existing snapshot on the underlying storage system. This field
+                    is immutable.
+                  type: string
+                volumeHandle:
+                  description: volumeHandle specifies the CSI "volume_id" of the volume
+                    from which a snapshot should be dynamically taken from. This field
+                    is immutable.
+                  type: string
+              type: object
+            volumeSnapshotClassName:
+              description: name of the VolumeSnapshotClass to which this snapshot
+                belongs.
+              type: string
+            volumeSnapshotRef:
+              description: volumeSnapshotRef specifies the VolumeSnapshot object to
+                which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                field must reference to this VolumeSnapshotContent's name for the
+                bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                object, name and namespace of the VolumeSnapshot object MUST be provided
+                for binding to happen. This field is immutable after creation. Required.
+              properties:
+                apiVersion:
+                  description: API version of the referent.
+                  type: string
+                fieldPath:
+                  description: 'If referring to a piece of an object instead of an
+                    entire object, this string should contain a valid JSON/Go field
+                    access statement, such as desiredState.manifest.containers[2].
+                    For example, if the object reference is to a container within
+                    a pod, this would take on a value like: "spec.containers{name}"
+                    (where "name" refers to the name of the container that triggered
+                    the event) or if no container name is specified "spec.containers[2]"
+                    (container with index 2 in this pod). This syntax is chosen only
+                    to have some well-defined way of referencing a part of an object.
+                    TODO: this design is not final and this field is subject to change
+                    in the future.'
+                  type: string
+                kind:
+                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                namespace:
+                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                  type: string
+                resourceVersion:
+                  description: 'Specific resourceVersion to which this reference is
+                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                  type: string
+                uid:
+                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                  type: string
+              type: object
+          required:
+          - deletionPolicy
+          - driver
+          - source
+          - volumeSnapshotRef
+          type: object
+        status:
+          description: status represents the current information of a snapshot.
+          properties:
+            creationTime:
+              description: creationTime is the timestamp when the point-in-time snapshot
+                is taken by the underlying storage system. In dynamic snapshot creation
+                case, this field will be filled in with the "creation_time" value
+                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
+                snapshot, this field will be filled with the "creation_time" value
+                returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                it. If not specified, it indicates the creation time is unknown. The
+                format of this field is a Unix nanoseconds time encoded as an int64.
+                On Unix, the command `date +%s%N` returns the current time in nanoseconds
+                since 1970-01-01 00:00:00 UTC.
+              format: int64
+              type: integer
+            error:
+              description: error is the latest observed error during snapshot creation,
+                if any.
+              properties:
+                message:
+                  description: 'message is a string detailing the encountered error
+                    during snapshot creation if specified. NOTE: message may be logged,
+                    and it should not contain sensitive information.'
+                  type: string
+                time:
+                  description: time is the timestamp when the error was encountered.
+                  format: date-time
+                  type: string
+              type: object
+            readyToUse:
+              description: readyToUse indicates if a snapshot is ready to be used
+                to restore a volume. In dynamic snapshot creation case, this field
+                will be filled in with the "ready_to_use" value returned from CSI
+                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
+                field will be filled with the "ready_to_use" value returned from the
+                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
+                this field will be set to "True". If not specified, it means the readiness
+                of a snapshot is unknown.
+              type: boolean
+            restoreSize:
+              description: restoreSize represents the complete size of the snapshot
+                in bytes. In dynamic snapshot creation case, this field will be filled
+                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
+                gRPC call. For a pre-existing snapshot, this field will be filled
+                with the "size_bytes" value returned from the CSI "ListSnapshots"
+                gRPC call if the driver supports it. When restoring a volume from
+                this snapshot, the size of the volume MUST NOT be smaller than the
+                restoreSize if it is specified, otherwise the restoration will fail.
+                If not specified, it indicates that the size is unknown.
+              format: int64
+              minimum: 0
+              type: integer
+            snapshotHandle:
+              description: snapshotHandle is the CSI "snapshot_id" of a snapshot on
+                the underlying storage system. If not specified, it indicates that
+                dynamic snapshot creation has either failed or it is still in progress.
+              type: string
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

deploy/k8s-1.27/snapshot-controller/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

@@ -0,0 +1,188 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/260"
+  creationTimestamp: null
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.readyToUse
+    description: Indicates if a snapshot is ready to be used to restore a volume.
+    name: ReadyToUse
+    type: boolean
+  - JSONPath: .spec.source.persistentVolumeClaimName
+    description: Name of the source PVC from where a dynamically taken snapshot will
+      be created.
+    name: SourcePVC
+    type: string
+  - JSONPath: .spec.source.volumeSnapshotContentName
+    description: Name of the VolumeSnapshotContent which represents a pre-provisioned
+      snapshot.
+    name: SourceSnapshotContent
+    type: string
+  - JSONPath: .status.restoreSize
+    description: Represents the complete size of the snapshot.
+    name: RestoreSize
+    type: string
+  - JSONPath: .spec.volumeSnapshotClassName
+    description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+    name: SnapshotClass
+    type: string
+  - JSONPath: .status.boundVolumeSnapshotContentName
+    description: The name of the VolumeSnapshotContent to which this VolumeSnapshot
+      is bound.
+    name: SnapshotContent
+    type: string
+  - JSONPath: .status.creationTime
+    description: Timestamp when the point-in-time snapshot is taken by the underlying
+      storage system.
+    name: CreationTime
+    type: date
+  - JSONPath: .metadata.creationTimestamp
+    name: Age
+    type: date
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    singular: volumesnapshot
+  preserveUnknownFields: false
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: VolumeSnapshot is a user's request for either creating a point-in-time
+        snapshot of a persistent volume, or binding to a pre-existing snapshot.
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: 'spec defines the desired characteristics of a snapshot requested
+            by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+            Required.'
+          properties:
+            source:
+              description: source specifies where a snapshot will be created from.
+                This field is immutable after creation. Required.
+              properties:
+                persistentVolumeClaimName:
+                  description: persistentVolumeClaimName specifies the name of the
+                    PersistentVolumeClaim object in the same namespace as the VolumeSnapshot
+                    object where the snapshot should be dynamically taken from. This
+                    field is immutable.
+                  type: string
+                volumeSnapshotContentName:
+                  description: volumeSnapshotContentName specifies the name of a pre-existing
+                    VolumeSnapshotContent object. This field is immutable.
+                  type: string
+              type: object
+            volumeSnapshotClassName:
+              description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass
+                requested by the VolumeSnapshot. If not specified, the default snapshot
+                class will be used if one exists. If not specified, and there is no
+                default snapshot class, dynamic snapshot creation will fail. Empty
+                string is not allowed for this field. TODO(xiangqian): a webhook validation
+                on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes'
+              type: string
+          required:
+          - source
+          type: object
+        status:
+          description: 'status represents the current information of a snapshot. NOTE:
+            status can be modified by sources other than system controllers, and must
+            not be depended upon for accuracy. Controllers should only use information
+            from the VolumeSnapshotContent object after verifying that the binding
+            is accurate and complete.'
+          properties:
+            boundVolumeSnapshotContentName:
+              description: 'boundVolumeSnapshotContentName represents the name of
+                the VolumeSnapshotContent object to which the VolumeSnapshot object
+                is bound. If not specified, it indicates that the VolumeSnapshot object
+                has not been successfully bound to a VolumeSnapshotContent object
+                yet. NOTE: Specified boundVolumeSnapshotContentName alone does not
+                mean binding       is valid. Controllers MUST always verify bidirectional
+                binding between       VolumeSnapshot and VolumeSnapshotContent to
+                avoid possible security issues.'
+              type: string
+            creationTime:
+              description: creationTime is the timestamp when the point-in-time snapshot
+                is taken by the underlying storage system. In dynamic snapshot creation
+                case, this field will be filled in with the "creation_time" value
+                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
+                snapshot, this field will be filled with the "creation_time" value
+                returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                it. If not specified, it indicates that the creation time of the snapshot
+                is unknown.
+              format: date-time
+              type: string
+            error:
+              description: error is the last observed error during snapshot creation,
+                if any. This field could be helpful to upper level controllers(i.e.,
+                application controller) to decide whether they should continue on
+                waiting for the snapshot to be created based on the type of error
+                reported.
+              properties:
+                message:
+                  description: 'message is a string detailing the encountered error
+                    during snapshot creation if specified. NOTE: message may be logged,
+                    and it should not contain sensitive information.'
+                  type: string
+                time:
+                  description: time is the timestamp when the error was encountered.
+                  format: date-time
+                  type: string
+              type: object
+            readyToUse:
+              description: readyToUse indicates if a snapshot is ready to be used
+                to restore a volume. In dynamic snapshot creation case, this field
+                will be filled in with the "ready_to_use" value returned from CSI
+                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
+                field will be filled with the "ready_to_use" value returned from the
+                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
+                this field will be set to "True". If not specified, it means the readiness
+                of a snapshot is unknown.
+              type: boolean
+            restoreSize:
+              anyOf:
+              - type: integer
+              - type: string
+              description: restoreSize represents the complete size of the snapshot
+                in bytes. In dynamic snapshot creation case, this field will be filled
+                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
+                gRPC call. For a pre-existing snapshot, this field will be filled
+                with the "size_bytes" value returned from the CSI "ListSnapshots"
+                gRPC call if the driver supports it. When restoring a volume from
+                this snapshot, the size of the volume MUST NOT be smaller than the
+                restoreSize if it is specified, otherwise the restoration will fail.
+                If not specified, it indicates that the size is unknown.
+              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+              x-kubernetes-int-or-string: true
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1beta1
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

deploy/k8s-1.27/snapshot-controller/rbac-snapshot-controller.yaml

@@ -0,0 +1,80 @@
+# RBAC file for the snapshot controller.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # rename if there are conflicts
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  # change the name also here if the ClusterRole gets renamed
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default # TODO: replace with the namespace you want for your controller
+  name: snapshot-controller-leaderelection
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: default # TODO: replace with the namespace you want for your controller
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: default # TODO: replace with the namespace you want for your controller
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+

deploy/k8s-1.27/snapshot-controller/setup-snapshot-controller.yaml

@@ -0,0 +1,26 @@
+# This YAML file shows how to deploy the snapshot controller
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+spec:
+  serviceName: "snapshot-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccount: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: quay.io/k8scsi/snapshot-controller:v2.1.1
+          args:
+            - "--v=5"
+            - "--leader-election=false"
+          imagePullPolicy: Always