Merge pull request #11 from MeinhardZhou/fix/mount_nvmf_error

fix: make example available

README.md

@@ -7,10 +7,10 @@ Currently it implements bare minimum of th [CSI spec](https://github.com/contain
 
 ## Requirements
 
-The CSI NVMf driver requires initiator and target kernel versions to be Linux kernel 5.0 or newer.
+The CSI NVMf driver requires initiator and target kernel versions to be **Linux kernel 5.0 or newer**.
 Before using this csi driver, you should create a NVMf remote disk on the target side and record traddr/trport/trtype/nqn/deviceuuid.
 
-## Modprobe Nvmf mod on Initiator/Target
+## Modprobe Nvmf mod on K8sNode
 
 ```
 # when use TCP as transport
@@ -33,27 +33,34 @@ $ go get github.com/rexray/gocsi/csc
 $ make
 ```
 
-### 2. Start NVMf driver
+### 2.1 Start NVMf driver
 
 ```
 $ ./output/nvmfplugin --endpoint tcp://127.0.0.1:10000 --nodeid CSINode
 ```
 
+### 2.2 Prepare nvmf kernel target
+
+Follow [guide to set up kernel target](doc/setup_kernel_nvmf_target.md) to deploy kernel nvmf storage service on localhost.
+
 ### 3.1 Get plugin info
 ```
 $ csc identity plugin-info --endpoint tcp://127.0.0.1:10000
 "csi.nvmf.com" "v1.0.0"
 ```
 ### 3.2 NodePublish a volume
+
+**The information here is what you used in step 2.2**
+
 ```
-$ export TargetTrAddr="NVMf Target Server IP (Ex: 192.168.122.18)"
+export TargetTrAddr="NVMf Target Server IP (Ex: 192.168.122.18)"
-$ export TargetTrPort="NVMf Target Server Ip Port (Ex: 49153)"
+export TargetTrPort="NVMf Target Server Ip Port (Ex: 49153)"
-$ export TargetTrType="NVMf Target Type (Ex: tcp | rdma)"
+export TargetTrType="NVMf Target Type (Ex: tcp | rdma)"
-$ export DeviceUUID="NVMf Target Device UUID (Ex: 58668891-c3e4-45d0-b90e-824525c16080)"
+export DeviceUUID="NVMf Target Device UUID (Ex: 58668891-c3e4-45d0-b90e-824525c16080)"
-$ export NQN="NVMf Target NQN"
+export NQN="NVMf Target NQN"
-$ csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nvmf --attrib targetTrAddr=$TargetTrAddr
+csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nvmf --vol-context targetTrAddr=$TargetTrAddr \
-                   --attrib targetTrPort=$TargetTrPort --attrib targetTrType=$TargetTrType
+                   --vol-context targetTrPort=$TargetTrPort --vol-context targetTrType=$TargetTrType \
-                   --attrib deviceUUID=$DeviceUUID --attrib nqn=$NQN nvmftestvol
+                   --vol-context deviceUUID=$DeviceUUID --vol-context nqn=$NQN nvmftestvol
 nvmftestvol
 ```
 You can find a new disk on /mnt/nvmf
@@ -82,7 +89,7 @@ $ kubectl delete -f deploy/kubenetes/
 ```
 
 ### 3.1 Create Storage Class(Dynamic Provisioning) 
-> NotSupport for controller not ready
+> **NotSupport Now**
 - Create
 ```
 $ kubectl create -f examples/kubernetes/example/storageclass.yaml
@@ -92,8 +99,9 @@ $ kubectl create -f examples/kubernetes/example/storageclass.yaml
 $ kubectl get sc
 ```
 
-### 3.2 Create PV(Static Provisioning)
+### 3.2 Create PV and PVC(Static Provisioning)
-- Create
+> **Supported**
+- Create Pv
 ```
 $ kubectl create -f examples/kubernetes/example/pv.yaml
 ```
@@ -101,6 +109,16 @@ $ kubectl create -f examples/kubernetes/example/pv.yaml
 ```
 $ kubectl get pv
 ```
+
+- Create Pvc
+```
+$ kubectl create -f exameples/kubernetes/example/pvc.yaml
+```
+- Check
+```
+$ kubectl get pvc
+```
+
 ### 4. Create Nginx Container
 - Create Deployment
 ```

deploy/kubernetes/csi-node-rbac.yaml

@@ -1,36 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: csi-nvmf-node
-  labels:
-    app: csi-nvmf
-    role: node
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: csi-nvmf-node
-  labels:
-    app: csi-nvmf
-    role: node
-rules:
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: csi-nvmf-node
-  labels:
-    app: csi-nvmf
-    role: node
-subjects:
-  - kind: ServiceAccount
-    name: csi-nvmf-node
-    namespace: default
-roleRef:
-  kind: ClusterRole
-  name: csi-nvmf-node
-  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/csi-nvmf-controller.yaml

@@ -0,0 +1,69 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-nvmf-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-nvmf-controller
+  template:
+    metadata:
+      labels:
+        app: csi-nvmf-controller
+    spec:
+      serviceAccount: csi-nvmf-controller-sa
+      containers:
+        - name:  csi-provisioner
+          image: quay.io/k8scsi/csi-provisioner:v1.3.0
+          imagePullPolicy: "IfNotPresent"
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=2"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:v1.2.0
+          imagePullPolicy: "IfNotPresent"
+          args:
+            - "--v=2"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=false"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+
+        - name: csi-nvmf-plugin
+          image: nvmfplugin:latest
+          imagePullPolicy: "IfNotPresent"
+          args:
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--IsControllerServer=true"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: volume-map
+              mountPath: /var/lib/kubelet/plugins/csi.nvmf.com/volumes
+              mountPropagation: "HostToContainer"
+
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: volume-map
+          hostPath:
+            path:  /var/lib/kubelet/plugins/csi.nvmf.com/volumes
+            type: DirectoryOrCreate
+
+

deploy/kubernetes/csi-nvmf-node.yaml

@@ -2,19 +2,19 @@ kind: DaemonSet
 apiVersion: apps/v1
 metadata:
   name: csi-nvmf-node
+  namespace: kube-system
 spec:
   selector:
     matchLabels:
-      app: csi-nvmf
+      app: csi-nvmf-node
-      role: node
   template:
     metadata:
       labels:
-        app: csi-nvmf
+        app: csi-nvmf-node
-        role: node
     spec:
-      serviceAccount: csi-nvmf-node
+      serviceAccount: csi-nvmf-node-sa
       hostNetwork: true
+      dnsPolicy: Default
       containers:
         - name: node-registrar
           image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
@@ -29,7 +29,7 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
           args:
-            - "--v=5"
+            - "--v=2"
             - "--csi-address=/csi/csi.sock"
             - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.nvmf.com/csi.sock"
           volumeMounts:
@@ -51,7 +51,7 @@ spec:
             - "--nodeid=$(NODE_ID)"
           env:
             - name: CSI_ENDPOINT
-              value: unix://var/lib/kubelet/plugins/csi.nvmf.com/csi.sock
+              value: unix:///var/lib/kubelet/plugins/csi.nvmf.com/csi.sock
             - name: NODE_ID
               valueFrom:
                 fieldRef:
@@ -62,17 +62,11 @@ spec:
             - name: pods-mount-dir
               mountPath: /var/lib/kubelet/pods
               mountPropagation: "Bidirectional"
-            - name: plugin-mount-dir
-              mountPath: /plugin
-              mountPropagation: "Bidirectional"
             - name: host-dev
               mountPath: /dev
               mountPropagation: "HostToContainer"
             - name: host-sys
               mountPath: /sys
-            - name: nvmf-tcp
-              mountPath: /usr/sbin/nvmf
-              subPath: nvmf
             - name: lib-modules
               mountPath: /lib/modules
               readOnly: true
@@ -81,10 +75,6 @@ spec:
           hostPath:
             path: /var/lib/kubelet/plugins/csi.nvmf.com
             type: DirectoryOrCreate
-        - name: plugin-mount-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins/csi-nvmfplugin
-            type: DirectoryOrCreate
         - name: registration-dir
           hostPath:
             path: /var/lib/kubelet/plugins_registry
@@ -96,9 +86,6 @@ spec:
         - name: host-dev
           hostPath:
             path: /dev
-        - name: nvmf-tcp
-          hostPath:
-            path: /usr/sbin
         - name: host-sys
           hostPath:
             path: /sys

deploy/kubernetes/csi-nvmf-rbac.yaml

@@ -0,0 +1,86 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-nvmf-controller-sa
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-nvmf-node-sa
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nvmf-external-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name:  nvmf-external-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-nvmf-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name:  nvmf-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nvmf-external-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nvmf-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-nvmf-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: nvmf-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io

doc/setup_kernel_nvmf_target.md

@@ -0,0 +1,61 @@
+This a simple guide to setup a nvmf target
+
+### 1. Modprobe nvmet-tcp
+``` bash
+modprobe nvmet-tcp
+```
+
+### 2. Prepare a storage backend device (file or block)
+
+```bash
+#file
+truncate --size=20G /tmp/nvmet_test.img
+
+#block is like /dev/nvme1n1
+```
+
+### 3.1 Create nvmf subsystem
+
+example subsystem name: nqn.2022-08.org.test-nvmf.example
+
+``` bash
+mkdir /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example
+echo 1 > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/attr_allow_any_host
+echo 0123456789abcdef > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/attr_serial
+```
+
+### 3.2 Create nvmf namespace
+
+``` bash
+mkdir /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1
+
+# file storage backend
+echo "/tmp/nvmet_test.img" > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_path
+
+# block storage backend
+echo "/dev/nvme1n1" > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_path
+
+echo 1 > /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/enable
+```
+
+### 3.3 Create nvmf port
+
+``` bash
+mkdir /sys/kernel/config/nvmet/ports/1
+echo ipv4 > /sys/kernel/config/nvmet/ports/1/addr_adrfam
+echo tcp > /sys/kernel/config/nvmet/ports/1/addr_trtype #target transport type is tcp
+echo "192.168.122.18" > /sys/kernel/config/nvmet/ports/1/addr_traddr # target addr is 192.168.122.18
+echo 49153 > /sys/kernel/config/nvmet/ports/1/addr_trsvcid # target port is 49153 (rdms should be 4420)
+```
+
+### 3.4 Link port and namespace
+
+``` bash
+ln -s /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/ /sys/kernel/config/nvmet/ports/1/subsystems/nqn.2022-08.org.test-nvmf.example
+```
+
+### 3.5 Record the DeviceUUID
+
+``` bash
+cat  /sys/kernel/config/nvmet/subsystems/nqn.2022-08.org.test-nvmf.example/namespaces/1/device_uuid
+```

examples/kubernetes/example/nginx.yaml

@@ -23,5 +23,5 @@ spec:
               name: nvmf-volume
       volumes:
         - name: nvmf-volume
-          persistentVolume:
+          persistentVolumeClaim:
-            claimName: pvc-example
+            claimName: csi-nvmf-pvc

examples/kubernetes/example/pv.yaml

@@ -1,18 +1,19 @@
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-	name: nvmfplugin-pv
+  name: csi-nvmf-pv
 spec:
-	storageClassName: csi.nvmf.com
+  storageClassName: cs-nvmf-sc
-	accessModes:
+  accessModes:
-	- ReadWriteOnce
+    - ReadWriteOnce
-	capacity:
+  capacity:
-	  storage: 20Gi
+    storage: 20Gi
-	csi:
+  csi:
-	  driver: NVMf
+    driver: csi.nvmf.com
-	  volumeAttributes:
+    volumeHandle: nvmf-data-id
-	  	targetTrAddr: "192.168.122.18"
+    volumeAttributes:
-	  	targetTrPort: "49153"
+      targetTrAddr: "192.168.122.18"
-	  	targetTrType: "tcp"
+      targetTrPort: "49153"
-	  	deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080"
+      targetTrType: "tcp"
-	  	nqn: "nqn.2021-07.org.test-nvmf.example"
+      deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080"
+      nqn: "nqn.2022-08.org.test-nvmf.example"

examples/kubernetes/example/pvc.yaml

@@ -1,11 +1,12 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: pvc-example
+  name: csi-nvmf-pvc
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: csi-nvmf-test
+  storageClassName: csi-nvmf-sc
   resources:
     requests:
-      storage: 20Gi
+      storage: 20Gi
+      

examples/kubernetes/example/storageclass.yaml

@@ -1,7 +1,7 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: csi-nvmf-test
+  name: csi-nvmf-sc
 provisioner: csi.nvmf.com
 reclaimPolicy: Delete
 allowVolumeExpansion: true

pkg/nvmf/const.go

@@ -17,7 +17,7 @@ package nvmf
 
 const (
 	NVMF_NQN_SIZE = 223
-	SYS_NVMF      = "/sys/class/nvmf"
+	SYS_NVMF      = "/sys/class/nvme"
 )
 
 // Here erron

pkg/nvmf/driver.go

@@ -59,11 +59,7 @@ func NewDriver(conf *GlobalConfig) *driver {
 }
 
 func (d *driver) Run(conf *GlobalConfig) {
-
+	d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{})
-	d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
-		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
-		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
-	})
 	d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
 		csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
 	})

pkg/nvmf/fabrics.go

@@ -52,7 +52,7 @@ func getNvmfConnector(nvmfInfo *nvmfDiskInfo) *Connector {
 // connector provides a struct to hold all of the needed parameters to make nvmf connection
 
 func _connect(argStr string) error {
-	file, err := os.OpenFile("/dev/nvmf-fabrics", os.O_RDWR, 0666)
+	file, err := os.OpenFile("/dev/nvme-fabrics", os.O_RDWR, 0666)
 	if err != nil {
 		klog.Errorf("Connect: open NVMf fabrics error: %v", err)
 		return err
@@ -155,7 +155,7 @@ func (c *Connector) Connect() (string, error) {
 	}
 
 	baseString := fmt.Sprintf("nqn=%s,transport=%s,traddr=%s,trsvcid=%s", c.TargetNqn, c.Transport, c.TargetAddr, c.TargetPort)
-	devicePath := strings.Join([]string{"/dev/disk/by-id/nvmf-uuid", c.DeviceUUID}, ".")
+	devicePath := strings.Join([]string{"/dev/disk/by-id/nvme-uuid", c.DeviceUUID}, ".")
 
 	// connect to nvmf disk
 	err := _connect(baseString)
@@ -165,7 +165,7 @@ func (c *Connector) Connect() (string, error) {
 	klog.Infof("Connect Volume %s success nqn: %s", c.VolumeID, c.TargetNqn)
 	retries := int(c.RetryCount / c.CheckInterval)
 	if exists, err := waitForPathToExist(devicePath, retries, int(c.CheckInterval), c.Transport); !exists {
-		klog.Errorf("connect nqn %s error %v, rollback", c.TargetNqn, err)
+		klog.Errorf("connect nqn %s error %v, rollback!!!", c.TargetNqn, err)
 		ret := disconnectByNqn(c.TargetNqn)
 		if ret < 0 {
 			klog.Errorf("rollback error !!!")

pkg/nvmf/nvmf.go

@@ -110,6 +110,10 @@ func AttachDisk(req *csi.NodePublishVolumeRequest, nm nvmfDiskMounter) (string,
 		klog.Errorf("AttachDisk: VolumeID %s failed to connect, Error: %v", req.VolumeId, err)
 		return "", err
 	}
+	if devicePath == "" {
+		klog.Errorf("AttachDisk: VolumeId %s return nil devicePath", req.VolumeId)
+		return "", fmt.Errorf("VolumeId %s return nil devicePath", req.VolumeId)
+	}
 	klog.Infof("AttachDisk: Volume %s successful connected, Device：%s", req.VolumeId, devicePath)
 
 	mntPath := nm.targetPath

pkg/nvmf/nvmf_utils.go

@@ -32,9 +32,7 @@ import (
 )
 
 func waitForPathToExist(devicePath string, maxRetries, intervalSeconds int, deviceTransport string) (bool, error) {
-	var err error
 	for i := 0; i < maxRetries; i++ {
-		err = nil
 		if deviceTransport == "tcp" {
 			exist := utils.IsFileExisting(devicePath)
 			if exist {
@@ -49,11 +47,11 @@ func waitForPathToExist(devicePath string, maxRetries, intervalSeconds int, devi
 		}
 		time.Sleep(time.Second * time.Duration(intervalSeconds))
 	}
-	return false, err
+	return false, fmt.Errorf("not found devicePath %s", devicePath)
 }
 
 func GetDeviceNameByVolumeID(volumeID string) (deviceName string, err error) {
-	volumeLinkPath := strings.Join([]string{"/dev/disk/by-id/nvmf-uuid", volumeID}, ".")
+	volumeLinkPath := strings.Join([]string{"/dev/disk/by-id/nvme-uuid", volumeID}, ".")
 	stat, err := os.Lstat(volumeLinkPath)
 	if err != nil {
 		if os.IsNotExist(err) {