feat: support block volumemode

Signed-off-by: Meinhard Zhou <zhouenhua@bytedance.com>

examples/kubernetes/block-example/nginx.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-block-test
+  labels:
+    app: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          ports:
+            - containerPort: 80
+          volumeDevices:
+            - devicePath: /dev/nvmf
+              name: nvmf-volume
+      volumes:
+        - name: nvmf-volume
+          persistentVolumeClaim:
+            claimName: csi-nvmf-pvc

examples/kubernetes/block-example/pv.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: csi-nvmf-pv-block
+spec:
+  storageClassName: csi-nvmf-sc-block
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Block
+  capacity:
+    storage: 20Gi
+  csi:
+    driver: csi.nvmf.com
+    volumeHandle: nvmf-data-id
+    volumeAttributes:
+      targetTrAddr: "192.168.122.18"
+      targetTrPort: "49153"
+      targetTrType: "tcp"
+      deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080"
+      nqn: "nqn.2022-08.org.test-nvmf.example"

examples/kubernetes/block-example/pvc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-nvmf-pvc-block
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Block
+  storageClassName: csi-nvmf-sc-block
+  resources:
+    requests:
+      storage: 20Gi
+      

examples/kubernetes/block-example/storageclass.yaml

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-nvmf-sc-block
+provisioner: csi.nvmf.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true

examples/kubernetes/fs-example/nginx.yaml

@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: nginx-block-test1
+  name: nginx-fs-test
   labels:
     app: nginx
 spec:
@@ -24,4 +24,4 @@ spec:
       volumes:
         - name: nvmf-volume
           persistentVolumeClaim:
-            claimName: csi-nvmf-pvc
+            claimName: csi-nvmf-pvc-fs

examples/kubernetes/fs-example/pv.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-  name: csi-nvmf-pv
+  name: csi-nvmf-pv-fs
 spec:
-  storageClassName: csi-nvmf-sc
+  storageClassName: csi-nvmf-sc-fs
   accessModes:
     - ReadWriteOnce
   capacity:

examples/kubernetes/fs-example/pvc.yaml

@@ -1,11 +1,11 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: csi-nvmf-pvc
+  name: csi-nvmf-pvc-fs
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: csi-nvmf-sc
+  storageClassName: csi-nvmf-sc-fs
   resources:
     requests:
       storage: 20Gi

examples/kubernetes/fs-example/storageclass.yaml

@@ -1,7 +1,7 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: csi-nvmf-sc
+  name: csi-nvmf-sc-fs
 provisioner: csi.nvmf.com
 reclaimPolicy: Delete
 allowVolumeExpansion: true

pkg/nvmf/const.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -31,7 +31,7 @@ const (
 	DefaultDriverServicePort = "12230"
 	DefaultDriverVersion     = "v1.0.0"
 
-	DefaultVolumeMapPath = "/var/lib/nvmf/volumes"
+	DefaultVolumeMapPath = "/var/lib/kubelet/plugins/csi.nvmf.com/volumes"
 )
 
 type GlobalConfig struct {

pkg/nvmf/fabrics.go

@@ -202,12 +202,12 @@ func persistConnectorFile(c *Connector, filePath string) error {
 
 }
 
-func removeConnectorFile(targetPath string) {
+func removeConnectorFile(filePath string) {
 	// todo: here maybe be attack for os.Remove can operate any file, fix?
-	if err := os.Remove(targetPath + ".json"); err != nil {
-		klog.Errorf("DetachDisk: Can't remove connector file: %s", targetPath)
+	if err := os.Remove(filePath); err != nil {
+		klog.Errorf("DetachDisk: Can't remove connector file: %s", filePath)
 	}
-	if err := os.RemoveAll(targetPath); err != nil {
+	if err := os.RemoveAll(filePath); err != nil {
 		klog.Errorf("DetachDisk: failed to remove mount path Error: %v", err)
 	}
 }

pkg/nvmf/nodeserver.go

@@ -18,6 +18,7 @@ package nvmf
 
 import (
 	"os"
+	"path"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-csi/csi-driver-nvmf/pkg/utils"
@@ -53,8 +54,7 @@ func (n *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCa
 }
 
 func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
-
-	// 1. check parameters
+	// Pre-check
 	if req.GetVolumeCapability() == nil {
 		return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume missing Volume Capability in req.")
 	}
@@ -67,18 +67,49 @@ func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish
 		return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume missing TargetPath in req.")
 	}
 
-	// 2. attachdisk
+	klog.Infof("VolumeID %s publish to targetPath %s.", req.GetVolumeId(), req.GetTargetPath())
+	// Connect remote disk
 	nvmfInfo, err := getNVMfDiskInfo(req)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "NodePublishVolume: get NVMf disk info from req err: %v", err)
 	}
-	diskMounter := getNVMfDiskMounter(nvmfInfo, req)
 
-	// attachDisk realize connect NVMf disk and mount to docker path
-	_, err = AttachDisk(req, *diskMounter)
+	connector := getNvmfConnector(nvmfInfo)
+	devicePath, err := connector.Connect()
+
+	connectorFilePath := path.Join(DefaultVolumeMapPath, req.GetVolumeId()+".json")
+
 	if err != nil {
-		klog.Errorf("NodePublishVolume: Attach volume %s with error: %s", req.VolumeId, err.Error())
-		return nil, err
+		klog.Errorf("VolumeID %s failed to connect, Error: %v", req.VolumeId, err)
+		return nil, status.Errorf(codes.Internal, "VolumeID %s failed to connect, Error: %v", req.VolumeId, err)
+	}
+	if devicePath == "" {
+		klog.Errorf("VolumeID %s connected, but return nil devicePath", req.VolumeId)
+		return nil, status.Errorf(codes.Internal, "VolumeID %s connected, but return nil devicePath", req.VolumeId)
+	}
+	klog.Infof("Volume %s successful connected, Device：%s", req.VolumeId, devicePath)
+	defer Rollback(err, func() {
+		connector.Disconnect()
+	})
+
+	err = persistConnectorFile(connector, connectorFilePath)
+	if err != nil {
+		klog.Errorf("failed to persist connection info: %v", err)
+		return nil, status.Errorf(codes.Internal, "VolumeID %s persist connection info error: %v", req.VolumeId, err)
+	}
+
+	// Attach disk to container path
+	if req.GetVolumeCapability().GetBlock() != nil && req.GetVolumeCapability().GetMount() != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "cannot have both block and mount access type")
+	}
+
+	defer Rollback(err, func() {
+		removeConnectorFile(connectorFilePath)
+	})
+
+	err = AttachDisk(req, devicePath)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "VolumeID %s attach error: %v", req.VolumeId, err)
 	}
 
 	return &csi.NodePublishVolumeResponse{}, nil
@@ -87,19 +118,36 @@ func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish
 func (n *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
 	klog.Infof("NodeUnpublishVolume: Starting unpublish volume, %s, %v", req.VolumeId, req)
 
+	// Pre-check
 	if req.VolumeId == "" {
 		return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume VolumeID must be provided")
 	}
 	if req.TargetPath == "" {
 		return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Staging TargetPath must be provided")
 	}
+
+	// Detach disk
 	targetPath := req.GetTargetPath()
-	err := DetachDisk(req.VolumeId, getNVMfDiskUnMounter(req), targetPath)
+	err := DetachDisk(req.VolumeId, targetPath)
 	if err != nil {
-		klog.Errorf("NodeUnpublishVolume: VolumeID: %s detachDisk err: %v", req.VolumeId, err)
+		klog.Errorf("VolumeID: %s detachDisk err: %v", req.VolumeId, err)
 		return nil, err
 	}
 
+	// Disconnect remote disk
+	connectorFilePath := path.Join(DefaultVolumeMapPath, req.GetVolumeId()+".json")
+	connector, err := GetConnectorFromFile(connectorFilePath)
+	if err != nil {
+		klog.Errorf("failed to get connector from path %s Error: %v", targetPath, err)
+		return nil, status.Errorf(codes.Internal, "failed to get connector from path %s Error: %v", targetPath, err)
+	}
+	err = connector.Disconnect()
+	if err != nil {
+		klog.Errorf("VolumeID: %s failed to disconnect, Error: %v", targetPath, err)
+		return nil, status.Errorf(codes.Internal, "failed to get connector from path %s Error: %v", targetPath, err)
+	}
+	removeConnectorFile(connectorFilePath)
+
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 

pkg/nvmf/nvmf.go

@@ -35,22 +35,6 @@ type nvmfDiskInfo struct {
 	Transport  string
 }
 
-type nvmfDiskMounter struct {
-	*nvmfDiskInfo
-	readOnly     bool
-	fsType       string
-	mountOptions []string
-	mounter      *mount.SafeFormatAndMount
-	exec         exec.Interface
-	targetPath   string
-	connector    *Connector
-}
-
-type nvmfDiskUnMounter struct {
-	mounter mount.Interface
-	exec    exec.Interface
-}
-
 func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) {
 	volName := req.GetVolumeId()
 
@@ -75,93 +59,83 @@ func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) {
 	}, nil
 }
 
-func getNVMfDiskMounter(nvmfInfo *nvmfDiskInfo, req *csi.NodePublishVolumeRequest) *nvmfDiskMounter {
-	readOnly := req.GetReadonly()
-	fsType := req.GetVolumeCapability().GetMount().GetFsType()
-	mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags()
+func AttachDisk(req *csi.NodePublishVolumeRequest, devicePath string) error {
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: exec.New()}
 
-	return &nvmfDiskMounter{
-		nvmfDiskInfo: nvmfInfo,
-		readOnly:     readOnly,
-		fsType:       fsType,
-		mountOptions: mountOptions,
-		mounter:      &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: exec.New()},
-		exec:         exec.New(),
-		targetPath:   req.GetTargetPath(),
-		connector:    getNvmfConnector(nvmfInfo),
+	targetPath := req.GetTargetPath()
+
+	if req.GetVolumeCapability().GetBlock() != nil {
+		_, err := os.Lstat(targetPath)
+		if os.IsNotExist(err) {
+			if err = makeFile(targetPath); err != nil {
+				return fmt.Errorf("failed to create target path, err: %s", err.Error())
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("failed to check if the target block file exist, err: %s", err.Error())
+		}
+
+		notMounted, err := mounter.IsLikelyNotMountPoint(targetPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return fmt.Errorf("error checking path %s for mount: %w", targetPath, err)
+			}
+			notMounted = true
+		}
+
+		if !notMounted {
+			klog.Infof("VolumeID: %s, Path: %s is already mounted, device: %s", req.VolumeId, targetPath, devicePath)
+			return nil
+		}
+
+		options := []string{""}
+		if err = mounter.Mount(devicePath, targetPath, "", options); err != nil {
+			klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, targetPath, options)
+			return fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, targetPath, options)
+		}
+
+	} else if req.GetVolumeCapability().GetMount() != nil {
+		notMounted, err := mounter.IsLikelyNotMountPoint(targetPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				klog.Errorf("AttachDisk: VolumeID: %s, Path %s is not exist, so create one.", req.GetVolumeId(), req.GetTargetPath())
+				if err = os.MkdirAll(targetPath, 0750); err != nil {
+					return fmt.Errorf("create target path: %v", err)
+				}
+				notMounted = true
+			} else {
+				return fmt.Errorf("check target path %v", err)
+			}
+		}
+
+		if !notMounted {
+			klog.Infof("AttachDisk: VolumeID: %s, Path: %s is already mounted.", req.GetVolumeId(), req.GetTargetPath())
+			return nil
+		}
+
+		fsType := req.GetVolumeCapability().GetMount().GetFsType()
+		readonly := req.GetReadonly()
+		mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags()
+		options := []string{""}
+		if readonly {
+			options = append(options, "ro")
+		} else {
+			options = append(options, "rw")
+		}
+		options = append(options, mountOptions...)
+
+		if err = mounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
+			klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, targetPath, options)
+			return fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, targetPath, options)
+		}
 	}
+	return nil
 }
 
-func getNVMfDiskUnMounter(req *csi.NodeUnpublishVolumeRequest) *nvmfDiskUnMounter {
-	return &nvmfDiskUnMounter{
-		mounter: mount.New(""),
-		exec:    exec.New(),
-	}
-}
+func DetachDisk(volumeID string, targetPath string) error {
+	mounter := mount.New("")
 
-func AttachDisk(req *csi.NodePublishVolumeRequest, nm nvmfDiskMounter) (string, error) {
-	if nm.connector == nil {
-		return "", fmt.Errorf("connector is nil")
-	}
-
-	// connect nvmf target disk
-	devicePath, err := nm.connector.Connect()
-	if err != nil {
-		klog.Errorf("AttachDisk: VolumeID %s failed to connect, Error: %v", req.VolumeId, err)
-		return "", err
-	}
-	if devicePath == "" {
-		klog.Errorf("AttachDisk: VolumeId %s return nil devicePath", req.VolumeId)
-		return "", fmt.Errorf("VolumeId %s return nil devicePath", req.VolumeId)
-	}
-	klog.Infof("AttachDisk: Volume %s successful connected, Device：%s", req.VolumeId, devicePath)
-
-	mntPath := nm.targetPath
-	klog.Infof("AttachDisk: MntPath: %s", mntPath)
-	notMounted, err := nm.mounter.IsLikelyNotMountPoint(mntPath)
-	if err != nil && !os.IsNotExist(err) {
-		return "", fmt.Errorf("Heuristic determination of mount point failed:%v", err)
-	}
-	if !notMounted {
-		klog.Infof("AttachDisk: VolumeID: %s, Path: %s is already mounted, device: %s", req.VolumeId, nm.targetPath, nm.DeviceUUID)
-		return "", nil
-	}
-
-	// pre to mount
-	if err := os.MkdirAll(mntPath, 0750); err != nil {
-		klog.Errorf("AttachDisk: failed to mkdir %s, error", mntPath)
-		return "", err
-	}
-
-	err = persistConnectorFile(nm.connector, mntPath+".json")
-	if err != nil {
-		klog.Errorf("AttachDisk: failed to persist connection info: %v", err)
-		klog.Errorf("AttachDisk: disconnecting volume and failing the publish request because persistence file are required for unpublish volume")
-		return "", fmt.Errorf("unable to create persistence file for connection")
-	}
-
-	// Tips: use k8s mounter to mount fs and only support "ext4"
-	var options []string
-	if nm.readOnly {
-		options = append(options, "ro")
-	} else {
-		options = append(options, "rw")
-	}
-	options = append(options, nm.mountOptions...)
-	err = nm.mounter.FormatAndMount(devicePath, mntPath, nm.fsType, options)
-	if err != nil {
-		klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, mntPath, options)
-		nm.connector.Disconnect()
-		removeConnectorFile(mntPath)
-		return "", fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, mntPath, options)
-	}
-
-	klog.Infof("AttachDisk: Successfully Mount Device %s to %s with options: %v", devicePath, mntPath, options)
-	return devicePath, nil
-}
-
-func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) error {
-	_, cnt, err := mount.GetDeviceNameFromMount(num.mounter, targetPath)
+	_, cnt, err := mount.GetDeviceNameFromMount(mounter, targetPath)
 	if err != nil {
 		klog.Errorf("nvmf detach disk: failed to get device from mnt: %s\nError: %v", targetPath, err)
 		return err
@@ -172,7 +146,7 @@ func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) erro
 		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", targetPath)
 		return nil
 	}
-	if err = num.mounter.Unmount(targetPath); err != nil {
+	if err = mounter.Unmount(targetPath); err != nil {
 		klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", targetPath, err)
 		return err
 	}
@@ -181,16 +155,5 @@ func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) erro
 		return nil
 	}
 
-	connector, err := GetConnectorFromFile(targetPath + ".json")
-	if err != nil {
-		klog.Errorf("DetachDisk: failed to get connector from path %s Error: %v", targetPath, err)
-		return err
-	}
-	err = connector.Disconnect()
-	if err != nil {
-		klog.Errorf("DetachDisk: VolumeID: %s failed to disconnect, Error: %v", volumeID, err)
-		return err
-	}
-	removeConnectorFile(targetPath)
 	return nil
 }

pkg/nvmf/nvmf_utils.go

@@ -97,3 +97,20 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h
 	}
 	return resp, err
 }
+
+func Rollback(err error, fc func()) {
+	if err != nil {
+		fc()
+	}
+}
+
+func makeFile(pathname string) error {
+	f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
+	defer f.Close()
+	if err != nil {
+		if !os.IsExist(err) {
+			return err
+		}
+	}
+	return nil
+}