diff --git a/examples/kubernetes/block-example/nginx.yaml b/examples/kubernetes/block-example/nginx.yaml new file mode 100644 index 0000000..0289060 --- /dev/null +++ b/examples/kubernetes/block-example/nginx.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-block-test + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeDevices: + - devicePath: /dev/nvmf + name: nvmf-volume + volumes: + - name: nvmf-volume + persistentVolumeClaim: + claimName: csi-nvmf-pvc \ No newline at end of file diff --git a/examples/kubernetes/block-example/pv.yaml b/examples/kubernetes/block-example/pv.yaml new file mode 100644 index 0000000..b9545bb --- /dev/null +++ b/examples/kubernetes/block-example/pv.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: csi-nvmf-pv-block +spec: + storageClassName: csi-nvmf-sc-block + accessModes: + - ReadWriteOnce + volumeMode: Block + capacity: + storage: 20Gi + csi: + driver: csi.nvmf.com + volumeHandle: nvmf-data-id + volumeAttributes: + targetTrAddr: "192.168.122.18" + targetTrPort: "49153" + targetTrType: "tcp" + deviceUUID: "58668891-c3e4-45d0-b90e-824525c16080" + nqn: "nqn.2022-08.org.test-nvmf.example" \ No newline at end of file diff --git a/examples/kubernetes/block-example/pvc.yaml b/examples/kubernetes/block-example/pvc.yaml new file mode 100644 index 0000000..ba0fe41 --- /dev/null +++ b/examples/kubernetes/block-example/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: csi-nvmf-pvc-block +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + storageClassName: csi-nvmf-sc-block + resources: + requests: + storage: 20Gi + \ No newline at end of file diff --git a/examples/kubernetes/block-example/storageclass.yaml b/examples/kubernetes/block-example/storageclass.yaml new file mode 100644 index 0000000..1ad0faf --- /dev/null +++ b/examples/kubernetes/block-example/storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-nvmf-sc-block +provisioner: csi.nvmf.com +reclaimPolicy: Delete +allowVolumeExpansion: true \ No newline at end of file diff --git a/examples/kubernetes/example/nginx.yaml b/examples/kubernetes/fs-example/nginx.yaml similarity index 88% rename from examples/kubernetes/example/nginx.yaml rename to examples/kubernetes/fs-example/nginx.yaml index 414b12d..22657af 100644 --- a/examples/kubernetes/example/nginx.yaml +++ b/examples/kubernetes/fs-example/nginx.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nginx-block-test1 + name: nginx-fs-test labels: app: nginx spec: @@ -24,4 +24,4 @@ spec: volumes: - name: nvmf-volume persistentVolumeClaim: - claimName: csi-nvmf-pvc \ No newline at end of file + claimName: csi-nvmf-pvc-fs \ No newline at end of file diff --git a/examples/kubernetes/example/pv.yaml b/examples/kubernetes/fs-example/pv.yaml similarity index 87% rename from examples/kubernetes/example/pv.yaml rename to examples/kubernetes/fs-example/pv.yaml index 2ee570a..a7ea0ba 100644 --- a/examples/kubernetes/example/pv.yaml +++ b/examples/kubernetes/fs-example/pv.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: csi-nvmf-pv + name: csi-nvmf-pv-fs spec: - storageClassName: csi-nvmf-sc + storageClassName: csi-nvmf-sc-fs accessModes: - ReadWriteOnce capacity: diff --git a/examples/kubernetes/example/pvc.yaml b/examples/kubernetes/fs-example/pvc.yaml similarity index 71% rename from examples/kubernetes/example/pvc.yaml rename to examples/kubernetes/fs-example/pvc.yaml index b506531..f879028 100644 --- a/examples/kubernetes/example/pvc.yaml +++ b/examples/kubernetes/fs-example/pvc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: csi-nvmf-pvc + name: csi-nvmf-pvc-fs spec: accessModes: - ReadWriteOnce - storageClassName: csi-nvmf-sc + storageClassName: csi-nvmf-sc-fs resources: requests: storage: 20Gi diff --git a/examples/kubernetes/example/storageclass.yaml b/examples/kubernetes/fs-example/storageclass.yaml similarity index 85% rename from examples/kubernetes/example/storageclass.yaml rename to examples/kubernetes/fs-example/storageclass.yaml index d4cb830..270a5e0 100644 --- a/examples/kubernetes/example/storageclass.yaml +++ b/examples/kubernetes/fs-example/storageclass.yaml @@ -1,7 +1,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: csi-nvmf-sc + name: csi-nvmf-sc-fs provisioner: csi.nvmf.com reclaimPolicy: Delete allowVolumeExpansion: true \ No newline at end of file diff --git a/pkg/nvmf/const.go b/pkg/nvmf/const.go index 3ed73cb..c429f14 100644 --- a/pkg/nvmf/const.go +++ b/pkg/nvmf/const.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -31,7 +31,7 @@ const ( DefaultDriverServicePort = "12230" DefaultDriverVersion = "v1.0.0" - DefaultVolumeMapPath = "/var/lib/nvmf/volumes" + DefaultVolumeMapPath = "/var/lib/kubelet/plugins/csi.nvmf.com/volumes" ) type GlobalConfig struct { diff --git a/pkg/nvmf/fabrics.go b/pkg/nvmf/fabrics.go index 7b0672f..0777eea 100644 --- a/pkg/nvmf/fabrics.go +++ b/pkg/nvmf/fabrics.go @@ -202,12 +202,12 @@ func persistConnectorFile(c *Connector, filePath string) error { } -func removeConnectorFile(targetPath string) { +func removeConnectorFile(filePath string) { // todo: here maybe be attack for os.Remove can operate any file, fix? - if err := os.Remove(targetPath + ".json"); err != nil { - klog.Errorf("DetachDisk: Can't remove connector file: %s", targetPath) + if err := os.Remove(filePath); err != nil { + klog.Errorf("DetachDisk: Can't remove connector file: %s", filePath) } - if err := os.RemoveAll(targetPath); err != nil { + if err := os.RemoveAll(filePath); err != nil { klog.Errorf("DetachDisk: failed to remove mount path Error: %v", err) } } diff --git a/pkg/nvmf/nodeserver.go b/pkg/nvmf/nodeserver.go index d104ce3..ea3966c 100644 --- a/pkg/nvmf/nodeserver.go +++ b/pkg/nvmf/nodeserver.go @@ -18,6 +18,7 @@ package nvmf import ( "os" + "path" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/csi-driver-nvmf/pkg/utils" @@ -53,8 +54,7 @@ func (n *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCa } func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - - // 1. check parameters + // Pre-check if req.GetVolumeCapability() == nil { return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume missing Volume Capability in req.") } @@ -67,18 +67,49 @@ func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume missing TargetPath in req.") } - // 2. attachdisk + klog.Infof("VolumeID %s publish to targetPath %s.", req.GetVolumeId(), req.GetTargetPath()) + // Connect remote disk nvmfInfo, err := getNVMfDiskInfo(req) if err != nil { return nil, status.Errorf(codes.Internal, "NodePublishVolume: get NVMf disk info from req err: %v", err) } - diskMounter := getNVMfDiskMounter(nvmfInfo, req) - // attachDisk realize connect NVMf disk and mount to docker path - _, err = AttachDisk(req, *diskMounter) + connector := getNvmfConnector(nvmfInfo) + devicePath, err := connector.Connect() + + connectorFilePath := path.Join(DefaultVolumeMapPath, req.GetVolumeId()+".json") + if err != nil { - klog.Errorf("NodePublishVolume: Attach volume %s with error: %s", req.VolumeId, err.Error()) - return nil, err + klog.Errorf("VolumeID %s failed to connect, Error: %v", req.VolumeId, err) + return nil, status.Errorf(codes.Internal, "VolumeID %s failed to connect, Error: %v", req.VolumeId, err) + } + if devicePath == "" { + klog.Errorf("VolumeID %s connected, but return nil devicePath", req.VolumeId) + return nil, status.Errorf(codes.Internal, "VolumeID %s connected, but return nil devicePath", req.VolumeId) + } + klog.Infof("Volume %s successful connected, Device:%s", req.VolumeId, devicePath) + defer Rollback(err, func() { + connector.Disconnect() + }) + + err = persistConnectorFile(connector, connectorFilePath) + if err != nil { + klog.Errorf("failed to persist connection info: %v", err) + return nil, status.Errorf(codes.Internal, "VolumeID %s persist connection info error: %v", req.VolumeId, err) + } + + // Attach disk to container path + if req.GetVolumeCapability().GetBlock() != nil && req.GetVolumeCapability().GetMount() != nil { + return nil, status.Errorf(codes.InvalidArgument, "cannot have both block and mount access type") + } + + defer Rollback(err, func() { + removeConnectorFile(connectorFilePath) + }) + + err = AttachDisk(req, devicePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "VolumeID %s attach error: %v", req.VolumeId, err) } return &csi.NodePublishVolumeResponse{}, nil @@ -87,19 +118,36 @@ func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublish func (n *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { klog.Infof("NodeUnpublishVolume: Starting unpublish volume, %s, %v", req.VolumeId, req) + // Pre-check if req.VolumeId == "" { return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume VolumeID must be provided") } if req.TargetPath == "" { return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Staging TargetPath must be provided") } + + // Detach disk targetPath := req.GetTargetPath() - err := DetachDisk(req.VolumeId, getNVMfDiskUnMounter(req), targetPath) + err := DetachDisk(req.VolumeId, targetPath) if err != nil { - klog.Errorf("NodeUnpublishVolume: VolumeID: %s detachDisk err: %v", req.VolumeId, err) + klog.Errorf("VolumeID: %s detachDisk err: %v", req.VolumeId, err) return nil, err } + // Disconnect remote disk + connectorFilePath := path.Join(DefaultVolumeMapPath, req.GetVolumeId()+".json") + connector, err := GetConnectorFromFile(connectorFilePath) + if err != nil { + klog.Errorf("failed to get connector from path %s Error: %v", targetPath, err) + return nil, status.Errorf(codes.Internal, "failed to get connector from path %s Error: %v", targetPath, err) + } + err = connector.Disconnect() + if err != nil { + klog.Errorf("VolumeID: %s failed to disconnect, Error: %v", targetPath, err) + return nil, status.Errorf(codes.Internal, "failed to get connector from path %s Error: %v", targetPath, err) + } + removeConnectorFile(connectorFilePath) + return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/pkg/nvmf/nvmf.go b/pkg/nvmf/nvmf.go index 18950de..46a3329 100644 --- a/pkg/nvmf/nvmf.go +++ b/pkg/nvmf/nvmf.go @@ -35,22 +35,6 @@ type nvmfDiskInfo struct { Transport string } -type nvmfDiskMounter struct { - *nvmfDiskInfo - readOnly bool - fsType string - mountOptions []string - mounter *mount.SafeFormatAndMount - exec exec.Interface - targetPath string - connector *Connector -} - -type nvmfDiskUnMounter struct { - mounter mount.Interface - exec exec.Interface -} - func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) { volName := req.GetVolumeId() @@ -75,93 +59,83 @@ func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) { }, nil } -func getNVMfDiskMounter(nvmfInfo *nvmfDiskInfo, req *csi.NodePublishVolumeRequest) *nvmfDiskMounter { - readOnly := req.GetReadonly() - fsType := req.GetVolumeCapability().GetMount().GetFsType() - mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags() +func AttachDisk(req *csi.NodePublishVolumeRequest, devicePath string) error { + mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: exec.New()} - return &nvmfDiskMounter{ - nvmfDiskInfo: nvmfInfo, - readOnly: readOnly, - fsType: fsType, - mountOptions: mountOptions, - mounter: &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: exec.New()}, - exec: exec.New(), - targetPath: req.GetTargetPath(), - connector: getNvmfConnector(nvmfInfo), + targetPath := req.GetTargetPath() + + if req.GetVolumeCapability().GetBlock() != nil { + _, err := os.Lstat(targetPath) + if os.IsNotExist(err) { + if err = makeFile(targetPath); err != nil { + return fmt.Errorf("failed to create target path, err: %s", err.Error()) + } + } + if err != nil { + return fmt.Errorf("failed to check if the target block file exist, err: %s", err.Error()) + } + + notMounted, err := mounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("error checking path %s for mount: %w", targetPath, err) + } + notMounted = true + } + + if !notMounted { + klog.Infof("VolumeID: %s, Path: %s is already mounted, device: %s", req.VolumeId, targetPath, devicePath) + return nil + } + + options := []string{""} + if err = mounter.Mount(devicePath, targetPath, "", options); err != nil { + klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, targetPath, options) + return fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, targetPath, options) + } + + } else if req.GetVolumeCapability().GetMount() != nil { + notMounted, err := mounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + klog.Errorf("AttachDisk: VolumeID: %s, Path %s is not exist, so create one.", req.GetVolumeId(), req.GetTargetPath()) + if err = os.MkdirAll(targetPath, 0750); err != nil { + return fmt.Errorf("create target path: %v", err) + } + notMounted = true + } else { + return fmt.Errorf("check target path %v", err) + } + } + + if !notMounted { + klog.Infof("AttachDisk: VolumeID: %s, Path: %s is already mounted.", req.GetVolumeId(), req.GetTargetPath()) + return nil + } + + fsType := req.GetVolumeCapability().GetMount().GetFsType() + readonly := req.GetReadonly() + mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags() + options := []string{""} + if readonly { + options = append(options, "ro") + } else { + options = append(options, "rw") + } + options = append(options, mountOptions...) + + if err = mounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil { + klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, targetPath, options) + return fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, targetPath, options) + } } + return nil } -func getNVMfDiskUnMounter(req *csi.NodeUnpublishVolumeRequest) *nvmfDiskUnMounter { - return &nvmfDiskUnMounter{ - mounter: mount.New(""), - exec: exec.New(), - } -} +func DetachDisk(volumeID string, targetPath string) error { + mounter := mount.New("") -func AttachDisk(req *csi.NodePublishVolumeRequest, nm nvmfDiskMounter) (string, error) { - if nm.connector == nil { - return "", fmt.Errorf("connector is nil") - } - - // connect nvmf target disk - devicePath, err := nm.connector.Connect() - if err != nil { - klog.Errorf("AttachDisk: VolumeID %s failed to connect, Error: %v", req.VolumeId, err) - return "", err - } - if devicePath == "" { - klog.Errorf("AttachDisk: VolumeId %s return nil devicePath", req.VolumeId) - return "", fmt.Errorf("VolumeId %s return nil devicePath", req.VolumeId) - } - klog.Infof("AttachDisk: Volume %s successful connected, Device:%s", req.VolumeId, devicePath) - - mntPath := nm.targetPath - klog.Infof("AttachDisk: MntPath: %s", mntPath) - notMounted, err := nm.mounter.IsLikelyNotMountPoint(mntPath) - if err != nil && !os.IsNotExist(err) { - return "", fmt.Errorf("Heuristic determination of mount point failed:%v", err) - } - if !notMounted { - klog.Infof("AttachDisk: VolumeID: %s, Path: %s is already mounted, device: %s", req.VolumeId, nm.targetPath, nm.DeviceUUID) - return "", nil - } - - // pre to mount - if err := os.MkdirAll(mntPath, 0750); err != nil { - klog.Errorf("AttachDisk: failed to mkdir %s, error", mntPath) - return "", err - } - - err = persistConnectorFile(nm.connector, mntPath+".json") - if err != nil { - klog.Errorf("AttachDisk: failed to persist connection info: %v", err) - klog.Errorf("AttachDisk: disconnecting volume and failing the publish request because persistence file are required for unpublish volume") - return "", fmt.Errorf("unable to create persistence file for connection") - } - - // Tips: use k8s mounter to mount fs and only support "ext4" - var options []string - if nm.readOnly { - options = append(options, "ro") - } else { - options = append(options, "rw") - } - options = append(options, nm.mountOptions...) - err = nm.mounter.FormatAndMount(devicePath, mntPath, nm.fsType, options) - if err != nil { - klog.Errorf("AttachDisk: failed to mount Device %s to %s with options: %v", devicePath, mntPath, options) - nm.connector.Disconnect() - removeConnectorFile(mntPath) - return "", fmt.Errorf("failed to mount Device %s to %s with options: %v", devicePath, mntPath, options) - } - - klog.Infof("AttachDisk: Successfully Mount Device %s to %s with options: %v", devicePath, mntPath, options) - return devicePath, nil -} - -func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) error { - _, cnt, err := mount.GetDeviceNameFromMount(num.mounter, targetPath) + _, cnt, err := mount.GetDeviceNameFromMount(mounter, targetPath) if err != nil { klog.Errorf("nvmf detach disk: failed to get device from mnt: %s\nError: %v", targetPath, err) return err @@ -172,7 +146,7 @@ func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) erro klog.Warningf("Warning: Unmount skipped because path does not exist: %v", targetPath) return nil } - if err = num.mounter.Unmount(targetPath); err != nil { + if err = mounter.Unmount(targetPath); err != nil { klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", targetPath, err) return err } @@ -181,16 +155,5 @@ func DetachDisk(volumeID string, num *nvmfDiskUnMounter, targetPath string) erro return nil } - connector, err := GetConnectorFromFile(targetPath + ".json") - if err != nil { - klog.Errorf("DetachDisk: failed to get connector from path %s Error: %v", targetPath, err) - return err - } - err = connector.Disconnect() - if err != nil { - klog.Errorf("DetachDisk: VolumeID: %s failed to disconnect, Error: %v", volumeID, err) - return err - } - removeConnectorFile(targetPath) return nil } diff --git a/pkg/nvmf/nvmf_utils.go b/pkg/nvmf/nvmf_utils.go index b09500c..71f419b 100644 --- a/pkg/nvmf/nvmf_utils.go +++ b/pkg/nvmf/nvmf_utils.go @@ -97,3 +97,20 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h } return resp, err } + +func Rollback(err error, fc func()) { + if err != nil { + fc() + } +} + +func makeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +}