From d4a279a377551ff7a931f14579210e67396d8fc4 Mon Sep 17 00:00:00 2001 From: "cheolho.kang" Date: Wed, 21 May 2025 10:04:23 +0900 Subject: [PATCH] feat: add Helm chart for NVMe-oF CSI Driver with storage class configurations Signed-off-by: cheolho.kang --- deploy/helm/csi-driver-nvmf/Chart.yaml | 6 + .../templates/csi-nvmf-controller.yaml | 79 +++++++++ .../templates/csi-nvmf-driver.yaml | 7 + .../templates/csi-nvmf-node.yaml | 119 +++++++++++++ .../templates/csi-nvmf-rbac.yaml | 167 ++++++++++++++++++ .../templates/storageclass.yaml | 11 ++ deploy/helm/csi-driver-nvmf/values.yaml | 13 ++ release-tools/build.make | 47 ++++- 8 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/csi-driver-nvmf/Chart.yaml create mode 100644 deploy/helm/csi-driver-nvmf/templates/csi-nvmf-controller.yaml create mode 100644 deploy/helm/csi-driver-nvmf/templates/csi-nvmf-driver.yaml create mode 100644 deploy/helm/csi-driver-nvmf/templates/csi-nvmf-node.yaml create mode 100644 deploy/helm/csi-driver-nvmf/templates/csi-nvmf-rbac.yaml create mode 100644 deploy/helm/csi-driver-nvmf/templates/storageclass.yaml create mode 100644 deploy/helm/csi-driver-nvmf/values.yaml diff --git a/deploy/helm/csi-driver-nvmf/Chart.yaml b/deploy/helm/csi-driver-nvmf/Chart.yaml new file mode 100644 index 0000000..a21bf0d --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: csi-driver-nvmf +description: A Helm chart for NVMe-oF CSI Driver +type: application +version: v1.0.0 +appVersion: v1.0.0 \ No newline at end of file diff --git a/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-controller.yaml b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-controller.yaml new file mode 100644 index 0000000..137e13e --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-controller.yaml @@ -0,0 +1,79 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nvmf-controller + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nvmf-controller + template: + metadata: + labels: + app: csi-nvmf-controller + spec: + serviceAccount: csi-nvmf-controller-sa + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + imagePullPolicy: "IfNotPresent" + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.csiDriver.verbosityLevel | default 2 }}" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + imagePullPolicy: "IfNotPresent" + args: + - "--v={{ .Values.csiDriver.verbosityLevel | default 2 }}" + - "--csi-address=$(ADDRESS)" + - "--leader-election=false" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-nvmf-plugin + image: {{ .Values.csiDriver.image.repository | default "nvmfplugin" }}:{{ .Values.csiDriver.image.tag | default "v1.0.0" }} + imagePullPolicy: "Always" + args: + - "--v={{ .Values.csiDriver.verbosityLevel | default 2 }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--IsControllerServer=true" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: volume-map + mountPath: /var/lib/kubelet/plugins/csi.nvmf.com/volumes + mountPropagation: "HostToContainer" + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + volumes: + - name: socket-dir + emptyDir: {} + - name: volume-map + hostPath: + path: /var/lib/kubelet/plugins/csi.nvmf.com/volumes + type: DirectoryOrCreate + - name: dev + hostPath: + path: /dev + - name: sys + hostPath: + path: /sys diff --git a/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-driver.yaml b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-driver.yaml new file mode 100644 index 0000000..c2501c1 --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-driver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.nvmf.com +spec: + attachRequired: false + podInfoOnMount: true \ No newline at end of file diff --git a/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-node.yaml b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-node.yaml new file mode 100644 index 0000000..bca9cee --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-node.yaml @@ -0,0 +1,119 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-nvmf-node + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: csi-nvmf-node + template: + metadata: + labels: + app: csi-nvmf-node + spec: + serviceAccount: csi-nvmf-node-sa + hostNetwork: true + dnsPolicy: Default + containers: + - name: node-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.2 + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.nvmf.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + args: + - "--v={{ .Values.csiDriver.verbosityLevel | default 2 }}" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + volumeMounts: + - name: registration-dir + mountPath: /registration + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + + - name: csi-nvmf-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .Values.csiDriver.image.repository | default "nvmfplugin" }}:{{ .Values.csiDriver.image.tag | default "v1.0.0" }} + imagePullPolicy: "Always" + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(NODE_ID)" + - "--v={{ .Values.csiDriver.verbosityLevel | default 2 }}" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/csi.nvmf.com/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi.nvmf.com + - name: csi-dir + mountPath: /var/lib/kubelet/plugins/kubernetes.io + mountPropagation: "Bidirectional" + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: run-nvmf-dir + mountPath: /run/nvmf + - name: host-dev + mountPath: /dev + mountPropagation: "HostToContainer" + - name: host-sys + mountPath: /sys + - name: lib-modules + mountPath: /lib/modules + readOnly: true + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.nvmf.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate + - name: csi-dir + hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: run-nvmf-dir + hostPath: + path: /run/nvmf + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules diff --git a/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-rbac.yaml b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-rbac.yaml new file mode 100644 index 0000000..0e17302 --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/templates/csi-nvmf-rbac.yaml @@ -0,0 +1,167 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nvmf-controller-sa + namespace: {{ .Release.Namespace }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nvmf-node-sa + namespace: {{ .Release.Namespace }} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-provisioner-binding +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-provisioner-role +subjects: + - kind: ServiceAccount + name: csi-nvmf-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: nvmf-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nvmf-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-nvmf-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: nvmf-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-csi-node-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nvmf-node-binding +subjects: + - kind: ServiceAccount + name: csi-nvmf-node-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: nvmf-csi-node-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nvmf-node-volumeattachment-binding +subjects: + - kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: nvmf-csi-node-volumeattachment-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-csi-node-volumeattachment-role +rules: + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] \ No newline at end of file diff --git a/deploy/helm/csi-driver-nvmf/templates/storageclass.yaml b/deploy/helm/csi-driver-nvmf/templates/storageclass.yaml new file mode 100644 index 0000000..d79e9ae --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/templates/storageclass.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageClass.name }} +parameters: + targetTrAddr: {{ .Values.storageClass.parameters.targetTrAddr | quote }} + targetTrPort: {{ .Values.storageClass.parameters.targetTrPort | quote }} + targetTrType: {{ .Values.storageClass.parameters.targetTrType | quote }} +provisioner: csi.nvmf.com +reclaimPolicy: Delete +allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} \ No newline at end of file diff --git a/deploy/helm/csi-driver-nvmf/values.yaml b/deploy/helm/csi-driver-nvmf/values.yaml new file mode 100644 index 0000000..50d9665 --- /dev/null +++ b/deploy/helm/csi-driver-nvmf/values.yaml @@ -0,0 +1,13 @@ +csiDriver: + verbosityLevel: 4 # (0-5): 0=ERROR, 1=WARNING, 2=INFO(기본값), 3=DEBUG, 4,5=TRACE + image: + repository: nvmfplugin + tag: v1.0.0 + +storageClass: + name: csi-nvmf-sc + allowVolumeExpansion: false + parameters: + targetTrAddr: "192.168.122.18,192.168.122.19" + targetTrPort: "49153,49154" + targetTrType: "tcp" diff --git a/release-tools/build.make b/release-tools/build.make index 39a3477..a31b716 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -15,7 +15,7 @@ # force the usage of /bin/bash instead of /bin/sh SHELL := /bin/bash -.PHONY: build-% build container-% container push-% push clean test +.PHONY: build-% build container-% container push-% push clean test helm-lint helm-package helm-install helm-upgrade helm-uninstall # A space-separated list of all commands in the repository, must be # set in main Makefile of a repository. @@ -330,3 +330,48 @@ test-boilerplate: test-logcheck: @ echo; echo "### $@:" @ ./release-tools/verify-logcheck.sh + +# Helm chart related targets +HELM_CHART_PATH ?= deploy/helm/csi-driver-nvmf +HELM_RELEASE_NAME ?= csi-nvmf +HELM_NAMESPACE ?= csi-nvmf + +# Lint Helm chart +.PHONY: helm-lint +helm-lint: + helm lint $(HELM_CHART_PATH) + +# Package Helm chart +.PHONY: helm-package +helm-package: + helm package $(HELM_CHART_PATH) + +# Install Helm chart +.PHONY: helm-install +helm-install: + helm install $(HELM_RELEASE_NAME) $(HELM_CHART_PATH) \ + --namespace $(HELM_NAMESPACE) \ + --create-namespace \ + -f $(HELM_CHART_PATH)/values.yaml + +# Upgrade Helm chart +.PHONY: helm-upgrade +helm-upgrade: + helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_PATH) \ + --namespace $(HELM_NAMESPACE) \ + -f $(HELM_CHART_PATH)/values.yaml + +# Uninstall Helm chart +.PHONY: helm-uninstall +helm-uninstall: + helm uninstall $(HELM_RELEASE_NAME) --namespace $(HELM_NAMESPACE) + +# Display Helm chart help +.PHONY: helm-help +helm-help: + @echo "Available Helm chart commands:" + @echo " make helm-lint - Lint Helm chart" + @echo " make helm-package - Package Helm chart" + @echo " make helm-install - Install Helm chart" + @echo " make helm-upgrade - Upgrade Helm chart" + @echo " make helm-uninstall - Uninstall Helm chart" \ No newline at end of file