diff --git a/README.md b/README.md index e6dd774..5495da6 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ Currently it implements bare minimum of th [CSI spec](https://github.com/contain ## Requirements -The CSI NVMf driver requires initiator and target kernel versions to be Linux kernel 5.0 or newer. +The CSI NVMf driver requires initiator and target kernel versions to be **Linux kernel 5.0 or newer**. Before using this csi driver, you should create a NVMf remote disk on the target side and record traddr/trport/trtype/nqn/deviceuuid. -## Modprobe Nvmf mod on Initiator/Target +## Modprobe Nvmf mod on K8sNode ``` # when use TCP as transport @@ -33,27 +33,34 @@ $ go get github.com/rexray/gocsi/csc $ make ``` -### 2. Start NVMf driver +### 2.1 Start NVMf driver ``` $ ./output/nvmfplugin --endpoint tcp://127.0.0.1:10000 --nodeid CSINode ``` +### 2.2 Prepare nvmf kernel target + +Follow [guide to set up kernel target](doc/setup_kernel_nvmf_target.md) to deploy kernel nvmf storage service on localhost. + ### 3.1 Get plugin info ``` $ csc identity plugin-info --endpoint tcp://127.0.0.1:10000 "csi.nvmf.com" "v1.0.0" ``` ### 3.2 NodePublish a volume + +**The information here is what you used in step 2.2** + ``` -$ export TargetTrAddr="NVMf Target Server IP (Ex: 192.168.122.18)" -$ export TargetTrPort="NVMf Target Server Ip Port (Ex: 49153)" -$ export TargetTrType="NVMf Target Type (Ex: tcp | rdma)" -$ export DeviceUUID="NVMf Target Device UUID (Ex: 58668891-c3e4-45d0-b90e-824525c16080)" -$ export NQN="NVMf Target NQN" -$ csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nvmf --attrib targetTrAddr=$TargetTrAddr - --attrib targetTrPort=$TargetTrPort --attrib targetTrType=$TargetTrType - --attrib deviceUUID=$DeviceUUID --attrib nqn=$NQN nvmftestvol +export TargetTrAddr="NVMf Target Server IP (Ex: 192.168.122.18)" +export TargetTrPort="NVMf Target Server Ip Port (Ex: 49153)" +export TargetTrType="NVMf Target Type (Ex: tcp | rdma)" +export DeviceUUID="NVMf Target Device UUID (Ex: 58668891-c3e4-45d0-b90e-824525c16080)" +export NQN="NVMf Target NQN" +csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nvmf --vol-context targetTrAddr=$TargetTrAddr \ + --vol-context targetTrPort=$TargetTrPort --vol-context targetTrType=$TargetTrType \ + --vol-context deviceUUID=$DeviceUUID --vol-context nqn=$NQN nvmftestvol nvmftestvol ``` You can find a new disk on /mnt/nvmf @@ -82,7 +89,7 @@ $ kubectl delete -f deploy/kubenetes/ ``` ### 3.1 Create Storage Class(Dynamic Provisioning) -> NotSupport for controller not ready +> **NotSupport Now** - Create ``` $ kubectl create -f examples/kubernetes/example/storageclass.yaml @@ -92,8 +99,9 @@ $ kubectl create -f examples/kubernetes/example/storageclass.yaml $ kubectl get sc ``` -### 3.2 Create PV(Static Provisioning) -- Create +### 3.2 Create PV and PVC(Static Provisioning) +> **Supported** +- Create Pv ``` $ kubectl create -f examples/kubernetes/example/pv.yaml ``` @@ -101,6 +109,16 @@ $ kubectl create -f examples/kubernetes/example/pv.yaml ``` $ kubectl get pv ``` + +- Create Pvc +``` +$ kubectl create -f exameples/kubernetes/example/pvc.yaml +``` +- Check +``` +$ kubectl get pvc +``` + ### 4. Create Nginx Container - Create Deployment ``` diff --git a/deploy/kubernetes/csi-node-rbac.yaml b/deploy/kubernetes/csi-node-rbac.yaml deleted file mode 100644 index 4abf015..0000000 --- a/deploy/kubernetes/csi-node-rbac.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-nvmf-node - labels: - app: csi-nvmf - role: node ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nvmf-node - labels: - app: csi-nvmf - role: node -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nvmf-node - labels: - app: csi-nvmf - role: node -subjects: - - kind: ServiceAccount - name: csi-nvmf-node - namespace: default -roleRef: - kind: ClusterRole - name: csi-nvmf-node - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/kubernetes/csi-nvmf-controller.yaml b/deploy/kubernetes/csi-nvmf-controller.yaml new file mode 100644 index 0000000..f01a8b6 --- /dev/null +++ b/deploy/kubernetes/csi-nvmf-controller.yaml @@ -0,0 +1,69 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nvmf-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nvmf-controller + template: + metadata: + labels: + app: csi-nvmf-controller + spec: + serviceAccount: csi-nvmf-controller-sa + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.3.0 + imagePullPolicy: "IfNotPresent" + args: + - "--csi-address=$(ADDRESS)" + - "--v=2" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.2.0 + imagePullPolicy: "IfNotPresent" + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election=false" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-nvmf-plugin + image: nvmfplugin:latest + imagePullPolicy: "IfNotPresent" + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--IsControllerServer=true" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: volume-map + mountPath: /var/lib/kubelet/plugins/csi.nvmf.com/volumes + mountPropagation: "HostToContainer" + + volumes: + - name: socket-dir + emptyDir: {} + - name: volume-map + hostPath: + path: /var/lib/kubelet/plugins/csi.nvmf.com/volumes + type: DirectoryOrCreate + + diff --git a/deploy/kubernetes/csi-nvmf-node.yaml b/deploy/kubernetes/csi-nvmf-node.yaml index 8ce622a..89c5bf0 100644 --- a/deploy/kubernetes/csi-nvmf-node.yaml +++ b/deploy/kubernetes/csi-nvmf-node.yaml @@ -2,19 +2,19 @@ kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-nvmf-node + namespace: kube-system spec: selector: matchLabels: - app: csi-nvmf - role: node + app: csi-nvmf-node template: metadata: labels: - app: csi-nvmf - role: node + app: csi-nvmf-node spec: - serviceAccount: csi-nvmf-node + serviceAccount: csi-nvmf-node-sa hostNetwork: true + dnsPolicy: Default containers: - name: node-registrar image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 @@ -29,7 +29,7 @@ spec: fieldRef: fieldPath: spec.nodeName args: - - "--v=5" + - "--v=2" - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.nvmf.com/csi.sock" volumeMounts: @@ -51,7 +51,7 @@ spec: - "--nodeid=$(NODE_ID)" env: - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi.nvmf.com/csi.sock + value: unix:///var/lib/kubelet/plugins/csi.nvmf.com/csi.sock - name: NODE_ID valueFrom: fieldRef: @@ -62,17 +62,11 @@ spec: - name: pods-mount-dir mountPath: /var/lib/kubelet/pods mountPropagation: "Bidirectional" - - name: plugin-mount-dir - mountPath: /plugin - mountPropagation: "Bidirectional" - name: host-dev mountPath: /dev mountPropagation: "HostToContainer" - name: host-sys mountPath: /sys - - name: nvmf-tcp - mountPath: /usr/sbin/nvmf - subPath: nvmf - name: lib-modules mountPath: /lib/modules readOnly: true @@ -81,10 +75,6 @@ spec: hostPath: path: /var/lib/kubelet/plugins/csi.nvmf.com type: DirectoryOrCreate - - name: plugin-mount-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-nvmfplugin - type: DirectoryOrCreate - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry @@ -96,9 +86,6 @@ spec: - name: host-dev hostPath: path: /dev - - name: nvmf-tcp - hostPath: - path: /usr/sbin - name: host-sys hostPath: path: /sys diff --git a/deploy/kubernetes/csi-nvmf-rbac.yaml b/deploy/kubernetes/csi-nvmf-rbac.yaml new file mode 100644 index 0000000..4e9a435 --- /dev/null +++ b/deploy/kubernetes/csi-nvmf-rbac.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nvmf-controller-sa + namespace: kube-system + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nvmf-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-provisioner-role +subjects: + - kind: ServiceAccount + name: csi-nvmf-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nvmf-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nvmf-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nvmf-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-nvmf-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nvmf-external-attacher-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/doc/setup_kernel_nvmf_target.md b/doc/setup_kernel_nvmf_target.md new file mode 100644 index 0000000..02cf0b8 --- /dev/null +++ b/doc/setup_kernel_nvmf_target.md @@ -0,0 +1,61 @@ +This a simple guide to setup a nvmf target + +### 1. Modprobe nvmet-tcp +``` bash +modprobe nvmet-tcp +``` + +### 2. Prepare a storage backend device (file or block) + +```bash +#file +truncate --size=20G /tmp/nvmet_test.img + +#block is like /dev/nvme1n1 +``` + +### 3.1 Create nvmf subsystem + +example subsystem name: nqn.2022-08.org.test-nvmf.example + +``` bash +mkdir /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example +echo 1 > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/attr_allow_any_host +echo 0123456789abcdef > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/attr_serial +``` + +### 3.2 Create nvmf namespace + +``` bash +mkdir /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1 + +# file storage backend +echo "/tmp/nvmet_test.img" > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_path + +# block storage backend +echo "/dev/nvme1n1" > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_path + +echo 1 > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/enable +``` + +### 3.3 Create nvmf port + +``` bash +mkdir /sys/kernel/config/nvmet/ports/1 +echo ipv4 > /sys/kernel/config/nvmet/ports/1/addr_adrfam +echo tcp > /sys/kernel/config/nvmet/ports/1/addr_trtype #target transport type is tcp +echo "192.168.122.18" > /sys/kernel/config/nvmet/ports/1/addr_traddr # target addr is 192.168.122.18 +echo 49153 > /sys/kernel/config/nvmet/ports/1/addr_trsvcid # target port is 49153 (rdms should be 4420) +``` + +### 3.4 Link port and namespace + +``` bash +ln -s /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/ /sys/kernel/config/nvmet/ports/1/subsystems/nqn.2022-08.org.test-nvmf.example +``` + +### 3.5 Record the DeviceUUID + +``` bash +cat /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_uuid +``` \ No newline at end of file diff --git a/examples/kubernetes/example/nginx.yaml b/examples/kubernetes/example/nginx.yaml index 7100ed0..414b12d 100644 --- a/examples/kubernetes/example/nginx.yaml +++ b/examples/kubernetes/example/nginx.yaml @@ -23,5 +23,5 @@ spec: name: nvmf-volume volumes: - name: nvmf-volume - persistentVolume: - claimName: pvc-example \ No newline at end of file + persistentVolumeClaim: + claimName: csi-nvmf-pvc \ No newline at end of file diff --git a/examples/kubernetes/example/pv.yaml b/examples/kubernetes/example/pv.yaml index 471dec1..1c3610d 100644 --- a/examples/kubernetes/example/pv.yaml +++ b/examples/kubernetes/example/pv.yaml @@ -1,18 +1,19 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: nvmfplugin-pv + name: csi-nvmf-pv spec: - storageClassName: csi.nvmf.com - accessModes: - - ReadWriteOnce - capacity: - storage: 20Gi - csi: - driver: NVMf - volumeAttributes: - targetTrAddr: "192.168.122.18" - targetTrPort: "49153" - targetTrType: "tcp" - deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080" - nqn: "nqn.2021-07.org.test-nvmf.example" \ No newline at end of file + storageClassName: cs-nvmf-sc + accessModes: + - ReadWriteOnce + capacity: + storage: 20Gi + csi: + driver: csi.nvmf.com + volumeHandle: nvmf-data-id + volumeAttributes: + targetTrAddr: "192.168.122.18" + targetTrPort: "49153" + targetTrType: "tcp" + deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080" + nqn: "nqn.2022-08.org.test-nvmf.example" \ No newline at end of file diff --git a/examples/kubernetes/example/pvc.yaml b/examples/kubernetes/example/pvc.yaml index bca54b9..b506531 100644 --- a/examples/kubernetes/example/pvc.yaml +++ b/examples/kubernetes/example/pvc.yaml @@ -1,11 +1,12 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pvc-example + name: csi-nvmf-pvc spec: accessModes: - ReadWriteOnce - storageClassName: csi-nvmf-test + storageClassName: csi-nvmf-sc resources: requests: - storage: 20Gi \ No newline at end of file + storage: 20Gi + \ No newline at end of file diff --git a/examples/kubernetes/example/storageclass.yaml b/examples/kubernetes/example/storageclass.yaml index 9731b36..d4cb830 100644 --- a/examples/kubernetes/example/storageclass.yaml +++ b/examples/kubernetes/example/storageclass.yaml @@ -1,7 +1,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: csi-nvmf-test + name: csi-nvmf-sc provisioner: csi.nvmf.com reclaimPolicy: Delete allowVolumeExpansion: true \ No newline at end of file diff --git a/pkg/nvmf/const.go b/pkg/nvmf/const.go index 42a74c8..3ed73cb 100644 --- a/pkg/nvmf/const.go +++ b/pkg/nvmf/const.go @@ -17,7 +17,7 @@ package nvmf const ( NVMF_NQN_SIZE = 223 - SYS_NVMF = "/sys/class/nvmf" + SYS_NVMF = "/sys/class/nvme" ) // Here erron diff --git a/pkg/nvmf/driver.go b/pkg/nvmf/driver.go index c8726fc..9d8b57e 100644 --- a/pkg/nvmf/driver.go +++ b/pkg/nvmf/driver.go @@ -59,11 +59,7 @@ func NewDriver(conf *GlobalConfig) *driver { } func (d *driver) Run(conf *GlobalConfig) { - - d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, - }) + d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{}) d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{ csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, }) diff --git a/pkg/nvmf/fabrics.go b/pkg/nvmf/fabrics.go index 5b19a7e..7b0672f 100644 --- a/pkg/nvmf/fabrics.go +++ b/pkg/nvmf/fabrics.go @@ -52,7 +52,7 @@ func getNvmfConnector(nvmfInfo *nvmfDiskInfo) *Connector { // connector provides a struct to hold all of the needed parameters to make nvmf connection func _connect(argStr string) error { - file, err := os.OpenFile("/dev/nvmf-fabrics", os.O_RDWR, 0666) + file, err := os.OpenFile("/dev/nvme-fabrics", os.O_RDWR, 0666) if err != nil { klog.Errorf("Connect: open NVMf fabrics error: %v", err) return err @@ -155,7 +155,7 @@ func (c *Connector) Connect() (string, error) { } baseString := fmt.Sprintf("nqn=%s,transport=%s,traddr=%s,trsvcid=%s", c.TargetNqn, c.Transport, c.TargetAddr, c.TargetPort) - devicePath := strings.Join([]string{"/dev/disk/by-id/nvmf-uuid", c.DeviceUUID}, ".") + devicePath := strings.Join([]string{"/dev/disk/by-id/nvme-uuid", c.DeviceUUID}, ".") // connect to nvmf disk err := _connect(baseString) @@ -165,7 +165,7 @@ func (c *Connector) Connect() (string, error) { klog.Infof("Connect Volume %s success nqn: %s", c.VolumeID, c.TargetNqn) retries := int(c.RetryCount / c.CheckInterval) if exists, err := waitForPathToExist(devicePath, retries, int(c.CheckInterval), c.Transport); !exists { - klog.Errorf("connect nqn %s error %v, rollback", c.TargetNqn, err) + klog.Errorf("connect nqn %s error %v, rollback!!!", c.TargetNqn, err) ret := disconnectByNqn(c.TargetNqn) if ret < 0 { klog.Errorf("rollback error !!!") diff --git a/pkg/nvmf/nvmf.go b/pkg/nvmf/nvmf.go index af44c60..af73431 100644 --- a/pkg/nvmf/nvmf.go +++ b/pkg/nvmf/nvmf.go @@ -110,6 +110,10 @@ func AttachDisk(req *csi.NodePublishVolumeRequest, nm nvmfDiskMounter) (string, klog.Errorf("AttachDisk: VolumeID %s failed to connect, Error: %v", req.VolumeId, err) return "", err } + if devicePath == "" { + klog.Errorf("AttachDisk: VolumeId %s return nil devicePath", req.VolumeId) + return "", fmt.Errorf("VolumeId %s return nil devicePath", req.VolumeId) + } klog.Infof("AttachDisk: Volume %s successful connected, Deviceļ¼š%s", req.VolumeId, devicePath) mntPath := nm.targetPath diff --git a/pkg/nvmf/nvmf_utils.go b/pkg/nvmf/nvmf_utils.go index 3f28a2e..b09500c 100644 --- a/pkg/nvmf/nvmf_utils.go +++ b/pkg/nvmf/nvmf_utils.go @@ -32,9 +32,7 @@ import ( ) func waitForPathToExist(devicePath string, maxRetries, intervalSeconds int, deviceTransport string) (bool, error) { - var err error for i := 0; i < maxRetries; i++ { - err = nil if deviceTransport == "tcp" { exist := utils.IsFileExisting(devicePath) if exist { @@ -49,11 +47,11 @@ func waitForPathToExist(devicePath string, maxRetries, intervalSeconds int, devi } time.Sleep(time.Second * time.Duration(intervalSeconds)) } - return false, err + return false, fmt.Errorf("not found devicePath %s", devicePath) } func GetDeviceNameByVolumeID(volumeID string) (deviceName string, err error) { - volumeLinkPath := strings.Join([]string{"/dev/disk/by-id/nvmf-uuid", volumeID}, ".") + volumeLinkPath := strings.Join([]string{"/dev/disk/by-id/nvme-uuid", volumeID}, ".") stat, err := os.Lstat(volumeLinkPath) if err != nil { if os.IsNotExist(err) {